From 0597ffd4f501355de27e075c1c855a28237b5742 Mon Sep 17 00:00:00 2001 From: Roel Aaij Date: Mon, 6 Jul 2020 19:49:47 +0200 Subject: [PATCH 1/2] Move Online/AllenIntegration to MooreOnline. --- Online/AllenIntegration/.gaudi_project_ignore | 0 Online/AllenIntegration/CMakeLists.txt | 43 --- Online/AllenIntegration/options/Allen.py | 71 ---- .../AllenIntegration/src/AllenApplication.cpp | 349 ------------------ .../AllenIntegration/src/AllenApplication.h | 96 ----- .../src/AllenConfiguration.cpp | 19 - .../AllenIntegration/src/AllenConfiguration.h | 33 -- 7 files changed, 611 deletions(-) delete mode 100644 Online/AllenIntegration/.gaudi_project_ignore delete mode 100644 Online/AllenIntegration/CMakeLists.txt delete mode 100755 Online/AllenIntegration/options/Allen.py delete mode 100644 Online/AllenIntegration/src/AllenApplication.cpp delete mode 100644 Online/AllenIntegration/src/AllenApplication.h delete mode 100644 Online/AllenIntegration/src/AllenConfiguration.cpp delete mode 100644 Online/AllenIntegration/src/AllenConfiguration.h diff --git a/Online/AllenIntegration/.gaudi_project_ignore b/Online/AllenIntegration/.gaudi_project_ignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Online/AllenIntegration/CMakeLists.txt b/Online/AllenIntegration/CMakeLists.txt deleted file mode 100644 index 5bfb33046ce..00000000000 --- a/Online/AllenIntegration/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Package: AllenIntegration -################################################################################ -gaudi_subdir(AllenIntegration v1r0) - -gaudi_depends_on_subdirs(Online/OnlineKernel - Online/GaudiOnline - Online/DIM - Online/RPC - Online/ZeroMQ - Online/Parsers - Dumpers/BinaryDumpers) - -find_package(cppgsl REQUIRED) -find_package(ZMQ REQUIRED) -find_package(Boost REQUIRED) -find_package(ROOT REQUIRED) -include_directories(SYSTEM ${Boost_INCLUDE_DIRS} - ${ZMQ_INCLUDE_DIRS} - ${CPPGSL_INCLUDE_DIR} - ${ROOT_INCLUDE_DIRS}) - -# find_package(MPI REQUIRED) -# include_directories(SYSTEM ${MPI_CXX_INCLUDE_PATH}) -# add_compile_definitions(HAVE_MPI) - -include_directories(${CMAKE_SOURCE_DIR}/main/include) - -gaudi_add_module(AllenIntegration - src/*.cpp - INCLUDE_DIRS OnlineBase ROOT Online/DIM GaudiKernel cppgsl Online/OnlineKernel Online/ZeroMQ - LINK_LIBRARIES GaudiKernel dim OnlineBase GaudiOnline ZMQ ZMQLib RPC Parsers - OnlineKernel AllenLib DAQEventLib DAQKernelLib GaudiAlgLib - PrKernel VPDetLib UTDetLib UTKernelLib - FTDetLib -lrt) -if(CMAKE_BUILD_TYPE STREQUAL Debug) - target_compile_definitions(AllenIntegration PRIVATE ALLEN_DEBUG) -endif() -target_compile_definitions(AllenIntegration PRIVATE ALLEN_DEBUG) - -if(TARGET_DEVICE STREQUAL "CPU" OR TARGET_DEVICE STREQUAL "CUDACLANG") - target_compile_definitions(AllenIntegration PRIVATE CPU) -endif() diff --git a/Online/AllenIntegration/options/Allen.py b/Online/AllenIntegration/options/Allen.py deleted file mode 100755 index 2f9769595af..00000000000 --- a/Online/AllenIntegration/options/Allen.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python2 -from Configurables import LHCbApp, CondDB, ApplicationMgr -from Configurables import DumpUTGeometry, DumpFTGeometry, DumpMuonTable -from Configurables import DumpMuonGeometry, DumpVPGeometry, AllenUpdater -from Configurables import DumpMagneticField, DumpBeamline, DumpUTLookupTables -from Configurables import AllenConfiguration -from Configurables import MonitorSvc -from Configurables import Online__Configuration as OnlineConfiguration - -app = LHCbApp( - DataType="Upgrade", - EvtMax=1000, - Simulation=True, - DDDBtag="dddb-20171122", - CondDBtag="sim-20180530-vc-md100") - -# Upgrade DBs -CondDB().Upgrade = True - -producers = [ - p(DumpToFile=False) - for p in (DumpVPGeometry, DumpUTGeometry, DumpFTGeometry, DumpMuonGeometry, - DumpMuonTable, DumpMagneticField, DumpBeamline, - DumpUTLookupTables) -] - -online_conf = OnlineConfiguration("Application") -online_conf.debug = False -online_conf.classType = 1 -online_conf.automatic = False -online_conf.monitorType = 'MonitorSvc' -online_conf.logDeviceType = 'RTL::Logger::LogDevice' -online_conf.logDeviceFormat = '%TIME%LEVEL%-8NODE: %-32PROCESS %-20SOURCE' -online_conf.OutputLevel = 3 -online_conf.IOOutputLevel = 3 - -allen_conf = AllenConfiguration() -allen_conf.EventsPerSlice = 1000 -allen_conf.NonStop = True -allen_conf.MPI = False -allen_conf.Receivers = "mlx5_0:1" -allen_conf.NThreads = 8 -allen_conf.NSlices = 16 -# allen_conf.Output = "tcp://192.168.1.101:35000" -# allen_conf.Device = "01:00.0" -allen_conf.Input = [ - "/scratch/raaij/mep/upgrade_mc_minbias_scifi_v5_pf3000.mep" -] -allen_conf.Device = "0" -allen_conf.OutputLevel = 2 - -monSvc = MonitorSvc('MonitorSvc') -monSvc.PartitionName = 'Allen' -monSvc.ExpandNameInfix = '' -monSvc.ExpandCounterServices = True -monSvc.UniqueServiceNames = True - -# Add the services that will produce the non-event-data -ApplicationMgr().ExtSvc += [ - monSvc, - AllenUpdater(OutputLevel=2), -] + producers - -# Some extra stuff for timing table -ApplicationMgr().EvtSel = "NONE" -ApplicationMgr().ExtSvc += [ - 'ToolSvc', 'AuditorSvc', 'AllenConfiguration', - 'Online::Configuration/Application', 'ZeroMQSvc' -] - -# gaudi = AppMgr() diff --git a/Online/AllenIntegration/src/AllenApplication.cpp b/Online/AllenIntegration/src/AllenApplication.cpp deleted file mode 100644 index b3eeea8e999..00000000000 --- a/Online/AllenIntegration/src/AllenApplication.cpp +++ /dev/null @@ -1,349 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#ifdef HAVE_MPI -#include -#endif - -#include "AllenConfiguration.h" -#include "AllenApplication.h" - -namespace { - using namespace std::string_literals; - - std::string resolveEnvVars(std::string s) - { - std::regex envExpr {"\\$\\{([A-Za-z0-9_]+)\\}"}; - std::smatch m; - while (std::regex_search(s, m, envExpr)) { - std::string rep; - System::getEnv(m[1].str(), rep); - s = s.replace(m[1].first - 2, m[1].second + 1, rep); - } - return s; - } -} // namespace - -/// Factory instantiation -DECLARE_COMPONENT(AllenApplication) - -/// Reset counters at start -void AllenApplication::monitor_t::reset() -{ - mepsIn = 0; - eventsOut = 0; -} - -/// Specialized constructor -AllenApplication::AllenApplication(Options opts) : OnlineApplication(opts) {} - -// Default destructor -AllenApplication::~AllenApplication() -{ - if (m_handle) { - dlclose(m_handle); - } -} - -/// Stop the application (RUNNING -> READY) -int AllenApplication::stop() -{ - fireIncident("DAQ_CANCEL"); - - m_zmqSvc->send(*m_allenControl, "STOP"); - - zmq::pollitem_t items[] = {{*m_allenControl, 0, zmq::POLLIN, 0}}; - m_zmqSvc->poll(&items[0], 1, -1); - if (items[0].revents & zmq::POLLIN) { - auto msg = m_zmqSvc->receive(*m_allenControl); - if (msg == "READY") { - m_logger->info("Allen event loop is stopped"); - } - else { - m_logger->error("Allen event loop failed to stop"); - return Online::ONLINE_ERROR; - } - } - - return OnlineApplication::stop(); -} - -/// Cancel the application: Cancel IO request/Event loop -int AllenApplication::cancel() { return 1; } - -/// Internal: Initialize the application (NOT_READY -> READY) -int AllenApplication::configureApplication() -{ - int ret = OnlineApplication::configureApplication(); - if (ret != Online::ONLINE_OK) return ret; - - // dlopen libAllenLib - m_handle = dlopen("libAllenLib.so", RTLD_LAZY); - if (!m_handle) { - m_logger->error("Failed to dlopen libAllenLib"); - return Online::ONLINE_ERROR; - } - - // reset errors - dlerror(); - // load the symbol - m_allenFun = (allen_t) dlsym(m_handle, "allen"); - const char* dlsym_error = dlerror(); - if (dlsym_error) { - m_logger->error("Failed to get 'allen' from libAllenLib"); - dlclose(m_handle); - return Online::ONLINE_ERROR; - } - - SmartIF sloc = app.as(); - - if (!m_config->monitorType.empty()) { - - m_monMEPs.reset(new Service("MEPs", sloc)); - m_monEvents.reset(new Service("Events", sloc)); - - m_monSvc = sloc->service(m_config->monitorType); - if (!m_monSvc.get()) { - m_logger->error("Cannot access monitoring service of type %s.", m_config->monitorType.c_str()); - return Online::ONLINE_ERROR; - } - m_monSvc->declareInfo("IN", m_monitor.mepsIn, "Number of MEPs received for processing", m_monMEPs); - m_monSvc->declareInfo("OUT", m_monitor.mepsDone, "Number of MEPs fully processed", m_monMEPs); - m_monSvc->declareInfo("OUT", m_monitor.eventsOut, "Number of events fully output", m_monEvents); - } - - auto config = sloc->service("AllenConfiguration/AllenConfiguration").as(); - if (!config.get()) { - m_logger->throwError("Failed to retrieve AllenConfiguration."); - return Online::ONLINE_ERROR; - } - m_allenConfig = config.get(); - - m_zmqSvc = sloc->service("ZeroMQSvc"); - if (!m_zmqSvc) { - m_logger->error("Failed to retrieve IZeroMQSvc."); - return Online::ONLINE_ERROR; - } - - SmartIF updater = sloc->service("AllenUpdater"); - if (!updater.get()) { - m_logger->error("Failed to retrieve AllenUpdater."); - return Online::ONLINE_ERROR; - } - m_updater = dynamic_cast(updater.get()); - if (updater == nullptr) { - m_logger->error("Failed to cast AllenUpdater"); - return Online::ONLINE_ERROR; - } - - if (m_allenConfig->withMPI.value()) { - auto success = initMPI(); - if (!success) { - m_logger->error("Failed to initialize MPI"); - return Online::ONLINE_ERROR; - } - } - - m_allenControl = m_zmqSvc->socket(zmq::PAIR); - m_allenControl->bind(m_controlConnection.c_str()); - - m_allenThread = std::thread {&AllenApplication::allenLoop, this}; - - zmq::pollitem_t items[] = {{*m_allenControl, 0, zmq::POLLIN, 0}}; - m_zmqSvc->poll(&items[0], 1, -1); - if (items[0].revents & zmq::POLLIN) { - auto msg = m_zmqSvc->receive(*m_allenControl); - if (msg == "READY") { - m_logger->info("Allen event loop is ready"); - } - } - - return ret; -} - -/// Internal: Finalize the application (READY -> NOT_READY) -int AllenApplication::finalizeApplication() -{ - m_zmqSvc->send(*m_allenControl, "RESET"); - - zmq::pollitem_t items[] = {{*m_allenControl, 0, zmq::POLLIN, 0}}; - m_zmqSvc->poll(&items[0], 1, -1); - if (items[0].revents & zmq::POLLIN) { - auto msg = m_zmqSvc->receive(*m_allenControl); - if (msg == "NOT_READY") { - m_logger->info("Allen event loop has exited"); - - m_allenThread.join(); - } - else { - m_logger->error("Allen event loop failed to exit"); - return Online::ONLINE_ERROR; - } - } - - if (m_monSvc.get()) { - m_monSvc->undeclareAll(m_monMEPs); - m_monSvc->undeclareAll(m_monEvents); - m_monSvc.reset(); - } - m_monMEPs.reset(); - m_monEvents.reset(); - return OnlineApplication::finalizeApplication(); -} - -/// Internal: Start the application (READY -> RUNNING) -int AllenApplication::startApplication() -{ - StatusCode sc = app->start(); - if (!sc.isSuccess()) { - return Online::ONLINE_ERROR; - } - - m_zmqSvc->send(*m_allenControl, "START"); - - zmq::pollitem_t items[] = {{*m_allenControl, 0, zmq::POLLIN, 0}}; - m_zmqSvc->poll(&items[0], 1, -1); - if (items[0].revents & zmq::POLLIN) { - auto msg = m_zmqSvc->receive(*m_allenControl); - if (msg == "RUNNING") { - m_logger->info("Allen event loop is running"); - } - else { - m_logger->error("Allen event loop failed to start"); - return Online::ONLINE_ERROR; - } - } - - fireIncident("DAQ_RUNNING"); - fireIncident("APP_RUNNING"); - return Online::ONLINE_OK; -} - -/// Pause the application (RUNNING -> READY) -int AllenApplication::pauseProcessing() -{ - m_logger->debug("Pause the application."); - return OnlineApplication::pauseProcessing(); -} - -/// Continue the application (PAUSED -> RUNNING ) -int AllenApplication::continueProcessing() -{ - m_logger->debug("Resume application processing."); - return OnlineApplication::continueProcessing(); -} - -bool AllenApplication::initMPI() -{ -#ifdef HAVE_MPI - // MPI initialization - auto len = name().length(); - int provided = 0; - m_mpiArgv = new char*[1]; - m_mpiArgv[0] = new char[len]; - ::strncpy(m_mpiArgv[0], name().c_str(), len); - MPI_Init_thread(&m_mpiArgc, &m_mpiArgv, MPI_THREAD_MULTIPLE, &provided); - if (provided != MPI_THREAD_MULTIPLE) { - m_logger->error("Failed to initialize MPI multi thread support."); - return false; - } - - // Communication size - int comm_size = 0; - MPI_Comm_size(MPI_COMM_WORLD, &comm_size); - if (comm_size > MPI::comm_size) { - std::string e = "This program requires at most "s + std::to_string(MPI::comm_size) + " processes."; - m_logger->error(e.c_str()); - return false; - } - - // MPI: Who am I? - MPI_Comm_rank(MPI_COMM_WORLD, &m_rank); - - if (m_rank != MPI::receiver) { - m_logger->error("AllenApplication can only function as MPI receiver."); - return false; - } - else { - return true; - } -#else - m_logger->error("MPI requested, but Allen was not built with MPI support."); - return false; -#endif -} - -void AllenApplication::allenLoop() -{ - - auto json = resolveEnvVars(m_allenConfig->json); - auto paramDir = resolveEnvVars(m_allenConfig->paramDir); - - //--events-per-slice 1000 --non-stop 1 --with-mpi $1:1 -c 0 -v 3 -t 8 -s 18 --output-file tcp://192.168.1.101:35000 - //--device 23:00.0 - std::map allen_options = {{"events-per-slice", std::to_string(m_allenConfig->eps.value())}, - {"non-stop", std::to_string(m_allenConfig->nonStop.value())}, - {"c", std::to_string(m_allenConfig->check.value())}, - {"v", std::to_string(6 - m_config->outputLevel())}, - {"t", std::to_string(m_allenConfig->nThreads.value())}, - {"geometry", paramDir}, - {"configuration", json}, - {"device", m_allenConfig->device.value()}}; - - if (!m_allenConfig->output.value().empty()) { - allen_options["output-file"] = m_allenConfig->output.value(); - } - - if (m_allenConfig->nSlices.value() != 0) { - allen_options["s"] = std::to_string(m_allenConfig->nSlices.value()); - } - - auto const& input = m_allenConfig->input.value(); - if (m_allenConfig->withMPI.value() == true) { - if (!m_allenConfig->receivers.value().empty()) { - allen_options["with-mpi"] = m_allenConfig->receivers.value(); - } - else { - allen_options["with-mpi"] = "1"; - } - } - else if (input.empty()) { - m_logger->throwError("No input files specified"); - } - else { - std::stringstream ss; - bool mep = false; - for (size_t i = 0; i < input.size(); ++i) { - if (i != 0) ss << ","; - if (input[i].find(".mep") != std::string::npos) mep = true; - ss << input[i]; - } - auto files = ss.str(); - allen_options[(mep ? "mep" : "mdf")] = files; - } - - m_allenFun(allen_options, m_updater, m_zmqSvc.get(), m_controlConnection); -} diff --git a/Online/AllenIntegration/src/AllenApplication.h b/Online/AllenIntegration/src/AllenApplication.h deleted file mode 100644 index 05f56359fd0..00000000000 --- a/Online/AllenIntegration/src/AllenApplication.h +++ /dev/null @@ -1,96 +0,0 @@ -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "AllenConfiguration.h" - -class AllenApplication : public Online::OnlineApplication { -public: - /// Structurte containing all monitoring items - struct monitor_t { - long mepsIn = 0; - long mepsDone = 0; - long eventsOut = 0; - monitor_t() = default; - virtual ~monitor_t() = default; - void reset(); - } m_monitor; - - // Specialized constructor - AllenApplication(Options opts); - // Default destructor - virtual ~AllenApplication(); - - /// Cancel the application: Cancel IO request/Event loop - int cancel() override; - - /// Internal: Initialize the application (NOT_READY -> READY) - int configureApplication() override; - /// Internal: Finalize the application (READY -> NOT_READY) - int finalizeApplication() override; - - /// Internal: Start the application (READY -> RUNNING) - int startApplication() override; - /// Stop the application (RUNNING -> READY) - int stop() override; - /// Pause the application (RUNNING -> PAUSED) - int pauseProcessing() override; - /// Continue the application (PAUSED -> RUNNING ) - int continueProcessing() override; - - // Main function running the Allen event loop - void allenLoop(); - - bool initMPI(); - -private: - /// Reference to the monitoring service - SmartIF m_monSvc; - - /// Handles to helper service to properly name burst counters - SmartIF m_monMEPs; - /// Handles to helper service to properly name event counters - SmartIF m_monEvents; - - // ZeroMQSvc - SmartIF m_zmqSvc; - - Allen::NonEventData::IUpdater* m_updater = nullptr; - AllenConfiguration const* m_allenConfig = nullptr; - - std::string m_controlConnection = "inproc://AllenApplicationControl"; - - // dlopen stuff to workaround segfault in genconf.exe - void* m_handle = nullptr; - typedef int ( - *allen_t)(std::map, Allen::NonEventData::IUpdater*, IZeroMQSvc* zmqSvc, std::string_view); - allen_t m_allenFun = nullptr; - - std::thread m_allenThread; - std::optional m_allenControl; - - char** m_mpiArgv = nullptr; - int m_mpiArgc = 1; - int m_rank = -1; -}; diff --git a/Online/AllenIntegration/src/AllenConfiguration.cpp b/Online/AllenIntegration/src/AllenConfiguration.cpp deleted file mode 100644 index 8d3af7610bd..00000000000 --- a/Online/AllenIntegration/src/AllenConfiguration.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "GaudiKernel/Service.h" -#include "AllenConfiguration.h" - -DECLARE_COMPONENT(AllenConfiguration) - -/// Query interfaces of Interface -StatusCode AllenConfiguration::queryInterface(const InterfaceID& riid, void** ppv) -{ - if (AllenConfiguration::interfaceID().versionMatch(riid)) { - *ppv = this; - addRef(); - return StatusCode::SUCCESS; - } - return Service::queryInterface(riid, ppv); -} - -AllenConfiguration::AllenConfiguration(std::string name, ISvcLocator* svcloc) : Service(name, svcloc) {} - -AllenConfiguration::~AllenConfiguration() {} diff --git a/Online/AllenIntegration/src/AllenConfiguration.h b/Online/AllenIntegration/src/AllenConfiguration.h deleted file mode 100644 index 4a7a431ce09..00000000000 --- a/Online/AllenIntegration/src/AllenConfiguration.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "GaudiKernel/Service.h" - -class AllenConfiguration : public Service { -public: - /// Retrieve interface ID - static const InterfaceID& interfaceID() - { - // Declaration of the interface ID. - static const InterfaceID iid("AllenConfiguration", 0, 0); - return iid; - } - - /// Query interfaces of Interface - StatusCode queryInterface(const InterfaceID& riid, void** ppv) override; - AllenConfiguration(std::string name, ISvcLocator* svcloc); - - ~AllenConfiguration(); - - Gaudi::Property eps {this, "EventsPerSlice", 1000}; - Gaudi::Property nonStop {this, "NonStop", true}; - Gaudi::Property withMPI {this, "MPI", true}; - Gaudi::Property receivers {this, "Receivers", ""}; - Gaudi::Property check {this, "CheckMC", false}; - Gaudi::Property nThreads {this, "NThreads", 8}; - Gaudi::Property nSlices {this, "NSlices", 16}; - Gaudi::Property output {this, "Output", ""}; - Gaudi::Property device {this, "Device", "0"}; - Gaudi::Property json {this, "JSON", "${ALLEN_PROJECT_ROOT}/configuration/constants/default.json"}; - Gaudi::Property paramDir {this, "ParamDir", "${ALLEN_PROJECT_ROOT}/input/detector_configuration/down"}; - Gaudi::Property> input {this, "Input"}; -}; -- GitLab From 0e7ccddf525708838be82378c46ecab98d82a063 Mon Sep 17 00:00:00 2001 From: Dorothea vom Bruch Date: Fri, 10 Jul 2020 17:03:28 +0200 Subject: [PATCH 2/2] remove Online dependencies --- CMakeLists.txt | 2 +- Rec/Allen/CMakeLists.txt | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8664418f3e8..88e5477f695 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ macro(allen_project) find_package(GaudiProject REQUIRED) #Declare project name and version - gaudi_project(Allen v0r9 USE Rec HEAD Online HEAD) + gaudi_project(Allen v0r9 USE Rec HEAD) #Always enable tests for gaudi builds set(BUILD_TESTS ON) diff --git a/Rec/Allen/CMakeLists.txt b/Rec/Allen/CMakeLists.txt index 09921f394d4..a73bf885028 100644 --- a/Rec/Allen/CMakeLists.txt +++ b/Rec/Allen/CMakeLists.txt @@ -55,8 +55,7 @@ include_directories(${CMAKE_SOURCE_DIR}/device/selections/Hlt1/include) gaudi_subdir(Allen v1r0) -gaudi_depends_on_subdirs(Online/OnlineKernel - Det/FTDet +gaudi_depends_on_subdirs(Det/FTDet Det/VPDet UT/UTDAQ UT/UTKernel @@ -81,7 +80,6 @@ gaudi_env(SET ALLEN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) gaudi_add_module(AllenWrapper src/*.cpp - INCLUDE_DIRS Online/OnlineKernel LINK_LIBRARIES AllenLib DAQEventLib DAQKernelLib GaudiAlgLib PrKernel VPDetLib UTDetLib UTKernelLib FTDetLib HltEvent) -- GitLab