diff --git a/Phys/FunctorCache/CMakeLists.txt b/Phys/FunctorCache/CMakeLists.txt index 50d88425a4b6354f014bab75c04ae4a180c80018..35481203aae1f37f35f86cf7be05462c244a7991 100644 --- a/Phys/FunctorCache/CMakeLists.txt +++ b/Phys/FunctorCache/CMakeLists.txt @@ -17,9 +17,9 @@ option(THOR_BUILD_TEST_FUNCTOR_CACHE "Build functor cache for THOR functors" ON) if(THOR_BUILD_TEST_FUNCTOR_CACHE) - # Import the cache creation module include(LoKiFunctorsCache) + # need to make sure the FunctorFactory is built set(cache_deps FunctorCore) # make sure GaudiConfig2 database has been correctly completed @@ -38,7 +38,10 @@ if(THOR_BUILD_TEST_FUNCTOR_CACHE) endforeach() endforeach() - # Also make sure that FunctorFactory is available + # Also make sure that FunctorFactory is available FIXME Since the new + # FunctorFactory in Rec!2699, I'm not sure if the directory is still needed + # but the comment never said "why" it was in the first place so it's hard + # to judge. (Same comment for SelAlgorithms...) if(NOT EXISTS ${PROJECT_SOURCE_DIR}/Phys/FunctorCore/CMakeLists.txt) message(FATAL_ERROR "Functor test cache can be build only if Phys/FunctorCore is present in the current project too") endif() @@ -53,27 +56,15 @@ if(THOR_BUILD_TEST_FUNCTOR_CACHE) # For now there is no need for a ThOr-specific alternative. set(LOKI_FUNCTORS_CACHE_POST_ACTION_OPTS) - loki_functors_cache(FunctorVectorTestCache - options/FlagInsideCacheGeneration.py - options/DisableLoKiCacheFunctors.py + loki_functors_cache(FunctorDatahandleTest options/SilenceErrors.py options/SuppressLogMessages.py - ${PROJECT_SOURCE_DIR}/Phys/FunctorCore/tests/options/test_vector_functors.py + options/ThOr_create_cache_opts.py + ${PROJECT_SOURCE_DIR}/Phys/FunctorCore/tests/options/functor_datahandle_test.py FACTORIES FunctorFactory LINK_LIBRARIES Rec::FunctorCoreLib DEPENDS ${cache_deps} - SPLIT 75 - ) - - loki_functors_cache(FunctorTestCache - options/DisableLoKiCacheFunctors.py - options/SilenceErrors.py - options/SuppressLogMessages.py - ${PROJECT_SOURCE_DIR}/Phys/FunctorCore/tests/options/test_functors.py - FACTORIES FunctorFactory - LINK_LIBRARIES Rec::FunctorCoreLib - DEPENDS ${cache_deps} - SPLIT 75 + SPLIT 2 ) endif(THOR_BUILD_TEST_FUNCTOR_CACHE) diff --git a/Phys/FunctorCache/options/DisableLoKiCacheFunctors.py b/Phys/FunctorCache/options/ThOr_create_cache_opts.py similarity index 71% rename from Phys/FunctorCache/options/DisableLoKiCacheFunctors.py rename to Phys/FunctorCache/options/ThOr_create_cache_opts.py index 6f9a917acef6754678d43d4a4f4ee247a01d6abc..11050a5b63920daa91c6837b1e1e47d6f67cc6f7 100644 --- a/Phys/FunctorCache/options/DisableLoKiCacheFunctors.py +++ b/Phys/FunctorCache/options/ThOr_create_cache_opts.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # @@ -9,5 +9,8 @@ # or submit itself to any jurisdiction. # ############################################################################### from Configurables import ApplicationMgr -ApplicationMgr().Environment['LOKI_DISABLE_CACHE'] = '1' -ApplicationMgr().Environment['LOKI_DISABLE_CLING'] = '1' + +ApplicationMgr().Environment['THOR_DISABLE_JIT'] = '1' +ApplicationMgr().Environment['THOR_DISABLE_CACHE'] = '1' +ApplicationMgr().Environment['THOR_JIT_EXTRA_ARGS'] = '' +ApplicationMgr().Environment['THOR_JIT_LIBDIR'] = '.' diff --git a/Phys/FunctorCore/CMakeLists.txt b/Phys/FunctorCore/CMakeLists.txt index 13d00f7f1dd79f92859204f1434901417a699d48..e8052e0e81de081e8f07111eac1284ddb03c026b 100644 --- a/Phys/FunctorCore/CMakeLists.txt +++ b/Phys/FunctorCore/CMakeLists.txt @@ -52,7 +52,6 @@ gaudi_add_module(FunctorCore FunctorCoreLib Gaudi::GaudiAlgLib Gaudi::GaudiKernel - ROOT::Core ) gaudi_add_executable(TestFunctors @@ -74,7 +73,170 @@ gaudi_add_executable(InstantiateFunctors TEST ) +# This target only exists to try and have a reliable way to figure out when to +# rebuild the preprocessed header +gaudi_add_executable(JIT_INCLUDES_TEST + SOURCES src/functor_jit_dummy/test_includes.cpp + LINK FunctorCoreLib + LHCb::PhysEvent + LHCb::TrackEvent + LHCb::MCEvent + Rec::ParticleCombinersLib +) + gaudi_install(PYTHON) gaudi_add_tests(QMTest) gaudi_add_tests(pytest ${CMAKE_CURRENT_SOURCE_DIR}/python) + + +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) + +set(preprocessed_header_name "preprocessed_functorfactory_header.ii") + +# When building the FunctorCache or when using a monobuild there is no install +# directory, thus we need to support running from the build directory and from +# the InstallArea. Executables like the below functor_jitter script and +# libraries are easy because the `CMAKE_CURRENT_BINARY_DIR` and the +# InstallArea are automatically in the `PATH` and `LD_LIBRARY_PATH` env +# variables. But to find the preprocessed header we need to play a small trick: +# The lhcb_env command sets an env variable for the build environment and the +# installed project, so that's what we do first and point to the InstallArea. +# Then we issue the command again but pass the PRIVATE flag which will only set +# the variable for the build directory thus overwriting the previously set env +# var for the build directory only. +lhcb_env(SET + FUNCTORFACTORY_PREPROCESSED_HEADER + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/${preprocessed_header_name}" +) +lhcb_env(PRIVATE SET + FUNCTORFACTORY_PREPROCESSED_HEADER + "${CMAKE_CURRENT_BINARY_DIR}/${preprocessed_header_name}" +) + +# # generate temporary file because I don't want to waste more time tyring to +# figure out how to freaking handle stupid whitespace in generator expressions +# and lists +file(GENERATE + OUTPUT "tmp_preprocessor.sh" + CONTENT "# auto generated +exec ${CMAKE_CXX_COMPILER} -x c++ -std=c++${GAUDI_CXX_STANDARD} \ +-D$>, -D> \ +${CMAKE_CXX_FLAGS} \ +${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} \ +-I$>,INCLUDE,/Rec/>, -I> \ +-isystem $>,EXCLUDE,/usr/include>,EXCLUDE,/Rec/>, -isystem > \ +-E ${CMAKE_CURRENT_SOURCE_DIR}/include/Functors/JIT_includes.h \ +-o ${preprocessed_header_name}" +) + + +# generate the preprocessed header which depends on JIT_INCLUDE_TEST +add_custom_command(OUTPUT ${preprocessed_header_name} + COMMAND sh tmp_preprocessor.sh + DEPENDS ${generated_header_name} "tmp_preprocessor.sh" + JIT_INCLUDES_TEST + ) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${preprocessed_header_name}" TYPE INCLUDE) + +# avoid "warning: style of line directive is a GCC extension" because we +# include a preprocessed header. Are there better solutions? We could first +# precompile the preprocessed header in initalize() and then use that pch... +# something for later +string(REPLACE " -pedantic" "" cxx_flags_without_pedantic ${CMAKE_CXX_FLAGS}) + +# the below logic assumes that `CMAKE_CXX_COMPILER` points to a compiler +# wrapper. These wrappers are created by the cmake logic defined in +# lcg-toolchains +file(READ "${CMAKE_CXX_COMPILER}" CMAKE_CXX_COMPILER_CONTENT) +string(REPLACE " \"$@\"" "" CMAKE_CXX_COMPILER_CONTENT ${CMAKE_CXX_COMPILER_CONTENT}) +string(STRIP ${CMAKE_CXX_COMPILER_CONTENT} CMAKE_CXX_COMPILER_CONTENT) + +# Specify the libraries a JIT compiled functor library will link against. The +# list here is defined based upon the includes present in +# `FunctorCore/include/JIT_includes.h` +set(JIT_LINK_LIBS "-lFunctorCore -lParticleCombiners -lTrackEvent -lPhysEvent -lMCEvent -lRecEvent -lHltEvent") + +file(GENERATE + OUTPUT "functor_jitter_tmp" + CONTENT "#!/usr/bin/env python +# Auto-generated script to create a jitter for the FunctorFactory +import os +import subprocess as sp +import sys +from multiprocessing import Pool + +header = os.environ['FUNCTORFACTORY_PREPROCESSED_HEADER'] + +if len(sys.argv) != 4: + raise Exception( + 'expect 4 arguments! e.g. functor_jitter {N_jobs} {source_directory} {output_lib_name}' + ) + +n_jobs = None if sys.argv[1] == '-1' else int(sys.argv[1]) +source_dir = sys.argv[2] +lib_name = sys.argv[3] +files = os.listdir(source_dir) +os.chdir(source_dir) + +# debug info is only needed for debugging or the throughput tests. Those jobs +# should set this env var otherwise if not set we explicitly force the debug +# level to zero to reduce memory and compilation time overhead of JIT by a +# factor of >2 +extra_args = os.environ.get('THOR_JIT_EXTRA_ARGS', '-g0') + +cmd = ''' +${CMAKE_CXX_COMPILER_CONTENT}''' + +my_pool = Pool(n_jobs) +return_codes = [] + +for file in files: + compile_cmd = cmd + ' -std=c++${GAUDI_CXX_STANDARD} \ + -D$>, -D> \ + ${cxx_flags_without_pedantic} \ + ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -fPIC \ + {2} -include {0} -c {1}'.format(header, file, extra_args) + + res = my_pool.apply_async(sp.call, (compile_cmd, ), {'shell': True}) + + return_codes.append(res) + +my_pool.close() +my_pool.join() + +if not all([r.successful() for r in return_codes]): + print('Nonzero exit codes in compilation jobs') + exit(1) + +# we know all our libs will be on the LD_LIBRARY_PATH so just point the linker there +my_env = os.environ.copy() +my_env['LIBRARY_PATH'] = my_env['LD_LIBRARY_PATH'] + +# include the CXX_FLAGS here again. This is for example needed when linking +# using clang as those flags contain the --gcc-tolchain= flag poiting clang to +# the gcc installation. +link_cmd = cmd + ' ${cxx_flags_without_pedantic} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} \ +-fPIC -shared {1} ${JIT_LINK_LIBS} -o {0} *.o'.format(lib_name, extra_args) + +exit(sp.call(link_cmd, shell=True, env=my_env)) +") + +# we don't yet have cmake 3.20 so file(GENERATE) doesn't accept permissions yet +# thus we add a proxy command that copies the generated file and makes it +# executable +add_custom_command(OUTPUT "functor_jitter" DEPENDS "functor_jitter_tmp" + COMMAND cp "functor_jitter_tmp" "functor_jitter" + COMMAND chmod a+x "functor_jitter" +) + + +install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/functor_jitter" TYPE BIN) + +add_custom_target(FunctorCoreJit ALL DEPENDS ${preprocessed_header_name} "functor_jitter") +# this is only here to handle dependencies of a FunctorCache outside Rec e.g. +# in Moore TODO this is technically a hack since `FunctorCoreJit` is only a +# runtime dependency but at the moment, there is no appropriate target for +# runtime dependencies to which `FunctorCoreJit` should be added. +add_dependencies(FunctorCore FunctorCoreJit) diff --git a/Phys/FunctorCore/include/Functors/Cache.h b/Phys/FunctorCore/include/Functors/Cache.h index 66f61a60202c2caa992703551a13e6a725e49ac9..b493803bd8a847d3cf9e7b36d95646969233ad02 100644 --- a/Phys/FunctorCore/include/Functors/Cache.h +++ b/Phys/FunctorCore/include/Functors/Cache.h @@ -29,5 +29,6 @@ namespace Functors::Cache { /** @brief Generate a Gaudi component name from a hash. */ - std::string id( HashType hash ); + std::string hashToStr( HashType hash ); + std::string hashToFuncName( HashType hash ); } // namespace Functors::Cache diff --git a/Phys/FunctorCore/include/Functors/Core.h b/Phys/FunctorCore/include/Functors/Core.h index 15603a5fd762545a91fc72802b3ae91e2357c967..b5f2832e941b4ee50b5a08c392cf35954fe4f620 100644 --- a/Phys/FunctorCore/include/Functors/Core.h +++ b/Phys/FunctorCore/include/Functors/Core.h @@ -495,11 +495,5 @@ namespace Functors { if ( !m_functor ) { throw std::runtime_error( "Empty Functor return type queried" ); } return m_functor->rtype(); } - - /** This is useful for the functor cache. - * - * @todo Give a more useful description... - */ - using Factory = typename ::Gaudi::PluginService::Factory; }; } // namespace Functors diff --git a/Phys/FunctorCore/include/Functors/IFactory.h b/Phys/FunctorCore/include/Functors/IFactory.h index 339d1e3074ffbaeb36e7a0293857d1fee290aa02..4a90dd695c239f402f211dcde838bd00efbd6f27 100644 --- a/Phys/FunctorCore/include/Functors/IFactory.h +++ b/Phys/FunctorCore/include/Functors/IFactory.h @@ -29,83 +29,69 @@ namespace Functors { * @brief Interface for turning strings into Functor instances. */ struct IFactory : extend_interfaces { - protected: - enum CompilationBehaviourBit { TryCache = 0x1, TryJIT = 0x2, ExceptionOnFailure = 0x4 }; - public: - /** Enum to flag whether the requested functor should be obtained from the - * functor cache, by JIT compilation, or either (the default). Also - * specifies whether or not failure should result in an exception or a - * null functor object being returned. Note that if all backends are - * disabled then a null functor object will always be returned and no - * exception will be raised. + /** */ - enum CompilationBehaviour { - CacheOrJIT = TryCache | TryJIT | ExceptionOnFailure, - CacheOnly = TryCache | ExceptionOnFailure, - JITOnly = TryJIT | ExceptionOnFailure, - QuietCacheOnly = TryCache, - QuietJITOnly = TryJIT - }; - - /** Default combination behaviour - */ - static constexpr CompilationBehaviour DefaultCompilationBehaviour = CacheOrJIT; - - protected: - using functor_base_t = std::unique_ptr; - constexpr static auto functor_base_t_str = "std::unique_ptr"; - /** Implementation method that gets an input/output-type-agnostic std::unique_ptr - * object from either cling or the cache. + /** + * @brief internal implementation method to register an input and + * output-type-agnostic std::unique_ptr object with the + * factory + * + * @param do_copy lambda created in register_functor, used to first + * cast and then copy the created functor into its + * registered destination location. + * @param functor_type string representation of the type of the functor + * @param desc ThOr::FunctorDesc object holding the functor code + * and "pretty" representation. */ - virtual functor_base_t get_impl( Gaudi::Algorithm* owner, std::string_view functor_type, - ThOr::FunctorDesc const& desc, CompilationBehaviour ) = 0; + virtual void do_register( std::function )> do_copy, + std::string_view functor_type, ThOr::FunctorDesc const& desc ) = 0; - public: - DeclareInterfaceID( IFactory, 1, 0 ); + DeclareInterfaceID( IFactory, 2, 0 ); - /** Factory method to get a C++ functor object from a string, either from - * the cache or using JIT compilation. + /** Factory method to register a C++ functor object to be created by this service. * * @param owner The algorithm that owns the functor, this is needed to * set up the functor's data dependencies correctly. - * @param desc ThOr::FunctorDesc object holding the functor code, list - * of headers required to compile it and "pretty" - * representation. - * @param compile CompilationBehaviour enum value specifying what should - * be tried when compiling this functor. By default the - * functor cache will be tried first, and the factory will - * fall back on JIT compilation. This does not override the - * global settings of the factory. - * @tparam FType Functor type that will be returned. This - * specifies precisely how the functor is instantiated. - * @return Functor object of the given type, may be empty. + * @param functor Functor of type FType that will be set by in + * the FunctorFactories start() call. + * Note: The functor is not usable until after Factory->start()!! + * @param desc ThOr::FunctorDesc object holding the functor code + * and "pretty" representation. */ template - FType get( Gaudi::Algorithm* owner, ThOr::FunctorDesc const& desc, - CompilationBehaviour compile = DefaultCompilationBehaviour ) { - auto any_functor = get_impl( owner, System::typeinfoName( typeid( FType ) ), desc, compile ); - if ( any_functor ) { // check the unique_ptr isn't empty - auto ftype_ptr = dynamic_cast( any_functor.get() ); // cast AnyFunctor* -> FType* (base -> derived) - if ( ftype_ptr ) { // check the AnyFunctor -> Functor conversion was OK - return std::move( *ftype_ptr ); // move the contents into the FType we return by value - } else { + void register_functor( Gaudi::Algorithm* owner, FType& functor, ThOr::FunctorDesc const& desc ) { + + // This lambda is a helper to perform the actual initalization of the + // algorithms' functor given the created AnyFunctor ptr from the + // FunctorFactory. It does 2 things: + // 1. It will remember the actual concrete type, e.g. + // Functors::Functor, we need to cast the pointer to before we + // can perform the copy. Otherwise we wold run into the classic object + // slicing problem as we only invoke the base class' move constructor + // 2. Remember the actual address we need to copy into (functor&). + // Note: we take ownership of the passed in pointer which is important because we + // are going to move the guts of the passed in object into the registered + // algorithm's functor + auto do_copy = [owner, &functor]( std::unique_ptr b ) { + auto ftype_ptr = dynamic_cast( b.get() ); // cast AnyFunctor* -> FType* (base -> derived) + + if ( !ftype_ptr ) { // This should only happen if you have a bug (e.g. you used a // SIMDWrapper type that has a different meaning depending on the // compilation flags in the stack/cling). We can't fix that at // runtime so let's just fail hard. throw GaudiException{"Failed to cast factory return type (" + - System::typeinfoName( typeid( decltype( *any_functor.get() ) ) ) + - ") to desired type (" + System::typeinfoName( typeid( FType ) ) + "), rtype is (" + - System::typeinfoName( any_functor->rtype() ) + ") and it " + - ( any_functor->wasJITCompiled() ? "was" : "was not" ) + " JIT compiled", - "Functors::IFactory::get( owner, desc, compile )", StatusCode::FAILURE}; + System::typeinfoName( typeid( decltype( *b ) ) ) + ") to desired type (" + + System::typeinfoName( typeid( FType ) ) + "), rtype is (" + + System::typeinfoName( b->rtype() ) + ") ", + "Functors::IFactory::register_functor( owner, functor, desc)", StatusCode::FAILURE}; } - } - // Return an empty FType object. This can happen if e.g. you disabled - // both cling and the cache, as is done during cache generation, so we - // should not abort the application... - return {}; + functor = std::move( *ftype_ptr ); + functor.bind( owner ); + }; + + do_register( do_copy, System::typeinfoName( typeid( FType ) ), desc ); } }; } // namespace Functors diff --git a/Phys/FunctorCore/include/Functors/JIT_includes.h b/Phys/FunctorCore/include/Functors/JIT_includes.h new file mode 100644 index 0000000000000000000000000000000000000000..363ea431a446dc80ab18935148206d77873a9a09 --- /dev/null +++ b/Phys/FunctorCore/include/Functors/JIT_includes.h @@ -0,0 +1,52 @@ +/*****************************************************************************\ +* (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration * +* * +* This software is distributed under the terms of the GNU General Public * +* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". * +* * +* In applying this licence, CERN does not waive the privileges and immunities * +* granted to it by virtue of its status as an Intergovernmental Organization * +* or submit itself to any jurisdiction. * +\*****************************************************************************/ + +/* + * + * This header only exists to capture all includes which are necessary for JIT + * compilation via the FunctorFactory. + * + * Do NOT include this in any cpp files!! + * + */ +#include +// Functors +#include "Functors/Adapters.h" +#include "Functors/Combination.h" +#include "Functors/Composite.h" +#include "Functors/Example.h" +#include "Functors/Filter.h" +#include "Functors/Function.h" +#include "Functors/MVA.h" +#include "Functors/NeutralLike.h" +#include "Functors/Particle.h" +#include "Functors/Simulation.h" +#include "Functors/TES.h" +#include "Functors/TrackLike.h" +// PhysEvent +#include "Event/Particle.h" +#include "Event/Particle_v2.h" +#include "Event/Vertex.h" +// TrackEvent +#include "Event/PrFittedForwardTracks.h" +#include "Event/Track_v1.h" +#include "Event/Track_v2.h" +#include "Event/Track_v3.h" +// TrackKernel +#include "TrackKernel/TrackCompactVertex.h" +// SelTools +#include "SelTools/MatrixNet.h" +#include "SelTools/SigmaNet.h" +// SelKernel +#include "SelKernel/ParticleCombination.h" +#include "SelKernel/VertexRelation.h" +// PrKernel +#include "PrKernel/PrSelection.h" diff --git a/Phys/FunctorCore/include/Functors/Particle.h b/Phys/FunctorCore/include/Functors/Particle.h index 5a16064f128c322a86c093ba680de46f490df783..223db783c5cda96ec3c873d838c57a6e87547581 100644 --- a/Phys/FunctorCore/include/Functors/Particle.h +++ b/Phys/FunctorCore/include/Functors/Particle.h @@ -12,10 +12,7 @@ #pragma once #include "Functors/Function.h" -#include "Functors/Utilities.h" #include "Kernel/IParticlePropertySvc.h" -#include "Kernel/ParticleProperty.h" -#include "LHCbMath/MatVec.h" #include "SelKernel/Utilities.h" /** @file Particle.h diff --git a/Phys/FunctorCore/include/Functors/Utilities.h b/Phys/FunctorCore/include/Functors/Utilities.h index 4befcbc05efa7189182f433e19c40f98265b3140..b5d0ebd3e4ce4b5225f2ee057094ae3b39f86393 100644 --- a/Phys/FunctorCore/include/Functors/Utilities.h +++ b/Phys/FunctorCore/include/Functors/Utilities.h @@ -112,7 +112,52 @@ namespace Functors::detail { void bind_helper( Algorithm* alg, std::index_sequence ) { static_assert( std::is_base_of_v, "You must include the full declaration of the owning algorithm type!" ); - ( std::get( m_handles ).emplace( m_tes_locs[Is], alg ), ... ); + + if ( alg->msgLevel( MSG::DEBUG ) ) { + alg->debug() << "Init of DataHandles of Functor: " + get_name( m_f ) << endmsg; + } + ( init_data_handle( std::get( m_handles ).emplace( m_tes_locs[Is], alg ), alg ), ... ); + } + + /** + * @brief Initialize a TES DataHandle and check that the owning algorithm + * was configured correctly and already holds our input in ExtraInputs + * + * For more info on the logic please see the detailed explanation of how + * functors obtain their data dependencies in the doc of the FunctorFactory. + * + * @param handle This handle will be initialized + * @param alg Algorithm/Tool which owns this functor + */ + template + void init_data_handle( DataObjectReadHandle& handle, Algorithm* alg ) { + if ( alg->msgLevel( MSG::DEBUG ) ) { + alg->debug() << " + " << handle.objKey() + << " (will call init(): " << ( alg->FSMState() == Gaudi::StateMachine::INITIALIZED ) << ")" + << endmsg; + } + + if ( alg->extraInputDeps().count( handle.objKey() ) == 0 ) { + throw GaudiException{"Usage of DataHandle[\"" + handle.objKey() + "\"] in Functor: " + get_name( m_f ) + + ", requires that owning algorithm " + alg->name() + + " contains this TES location inside the ExtraInputs property. This is likely a " + "Configuration/PyConf bug!", + get_name( m_f ), StatusCode::FAILURE}; + } + + // DataObjectReadHandle has a protected `init()` so we need to call it + // through it's base class. This is the same thing Gaudi::Algorithm does in + // sysInitialize(). We do it here because this DataHandle is created inside + // start(), at which point the step of initializing the handles of an + // algorithm has already happened. + // !! Exception !! if we are getting this functor from the cache then we + // are already creating it in intialize(), and we need to skip the init() + // call as it's also done in the sysInitialize() of the algorithm and it is + // apparently forbidden to call init() twice on a DataHandle which is + // checked via an assert in DataObjectHandleBase->init(). So we only run + // init() here if the algorithm is already in an INITIALIZEDD state which + // means this construction is happening inside start() + if ( alg->FSMState() == Gaudi::StateMachine::INITIALIZED ) { static_cast( &handle )->init(); } } /** Make a tuple of references to the result of dereferencing each diff --git a/Phys/FunctorCore/include/Functors/with_functor_maps.h b/Phys/FunctorCore/include/Functors/with_functor_maps.h index f613a45e15fa72853e07798d2f06275cc495a478..7b157c8267e794255bcbfabbbfd584e8a168acd4 100644 --- a/Phys/FunctorCore/include/Functors/with_functor_maps.h +++ b/Phys/FunctorCore/include/Functors/with_functor_maps.h @@ -36,21 +36,12 @@ class with_functor_maps : public Functors::detail::with_functor_factory template void decode() { - using TagType = boost::mp11::mp_at_c; - using FunctorType = boost::mp11::mp_at_c; // This is the {nickname: decoded_functor} map we want to populate auto& functor_map = std::get( m_functors ); // Clean it up each time in case we re-decode functor_map.clear(); for ( auto const& [func_name, func_desc] : m_properties.template get() ) { - // Local copy we can add extra headers to - ThOr::FunctorDesc proxy{func_desc}; - if constexpr ( Functors::detail::has_extra_headers_v ) { - for ( auto const& h : TagType::ExtraHeaders ) { proxy.headers.emplace_back( h ); } - } - // Decode the functor - functor_map[func_name] = this->getFunctorFactory().template get( - this, proxy, Functors::detail::get_compilation_behaviour_v ); + this->getFunctorFactory().register_functor( this, functor_map[func_name], func_desc ); } } diff --git a/Phys/FunctorCore/include/Functors/with_functors.h b/Phys/FunctorCore/include/Functors/with_functors.h index 4ef70af701821c75d279094170c0cda43cd003ee..59041416a87eb4fde9a1673293ae26ff419ccee9 100644 --- a/Phys/FunctorCore/include/Functors/with_functors.h +++ b/Phys/FunctorCore/include/Functors/with_functors.h @@ -31,21 +31,6 @@ namespace Functors::detail { template inline constexpr bool has_extra_headers_v = has_extra_headers::value; - /** @brief Check if the given type has a static member called Compilation - */ - template - struct get_compilation_behaviour { - static constexpr auto value = IFactory::DefaultCompilationBehaviour; - }; - - template - struct get_compilation_behaviour { - static constexpr auto value = T::Compilation; - }; - - template - inline constexpr auto get_compilation_behaviour_v = get_compilation_behaviour::value; - /** Type that we use to tag whether or not the functor factory service handle * has been added to a class. */ @@ -147,16 +132,9 @@ private: template void decode() { - using TagType = std::tuple_element_t; - using FunctorType = std::tuple_element_t; // Make a copy, as we might need to add headers to it - ThOr::FunctorDesc proxy = m_properties.template get(); - // Add extra headers if needed - if constexpr ( Functors::detail::has_extra_headers_v ) { - for ( auto const& h : TagType::ExtraHeaders ) { proxy.headers.emplace_back( h ); } - } - std::get( m_functors ) = this->getFunctorFactory().template get( - this, proxy, Functors::detail::get_compilation_behaviour_v ); + // FIXME note for cleanup of REGISTER_HEADER and similar header bookeeping + this->getFunctorFactory().register_functor( this, std::get( m_functors ), m_properties.template get() ); } // Storage for the decoded functors diff --git a/Phys/FunctorCore/include/Functors/with_output_tree.h b/Phys/FunctorCore/include/Functors/with_output_tree.h index aa80f5d2bd6a909eeff3fbd37ed19bddd304ec0c..d84be1272a4ac34fc6034dfee45903a6c2a70f5e 100644 --- a/Phys/FunctorCore/include/Functors/with_output_tree.h +++ b/Phys/FunctorCore/include/Functors/with_output_tree.h @@ -173,17 +173,21 @@ struct with_output_tree : public Functors::detail::with_output_tree::mixin_base< using mixin_base::mixin_base; StatusCode initialize() override { - // Delegate to the base class method, this makes sure our functors are - // available. - auto sc = mixin_base::initialize(); // Open the ROOT file and create the TTree m_root_file.reset( TFile::Open( m_root_file_name.value().c_str(), "recreate" ) ); m_root_file->cd(); // m_root_tree gets managed by m_root_file, this isn't a dangling pointer m_root_tree = new TTree( m_root_tree_name.value().c_str(), "" ); + return mixin_base::initialize(); + } + + StatusCode start() override { // Set up our vectors of branch-filling helpers that go with those functors + // we can not call this in initialize() because the current implementation + // relies on calling functor->rtype() thus we need to wait unilt after the + // FunctorFactory's start() call. ( initialize(), ... ); - return sc; + return mixin_base::start(); } StatusCode finalize() override { diff --git a/Phys/FunctorCore/python/Functors/utils.py b/Phys/FunctorCore/python/Functors/utils.py deleted file mode 100644 index cda3c777022cd474a95d83fbe2f9143787a5d22a..0000000000000000000000000000000000000000 --- a/Phys/FunctorCore/python/Functors/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -############################################################################### -# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -def pack_dict(input_dict, wrap=None): - """Given a string-keyed BoundFunctor-valued dictionary, pack that into a - dictionary-of-vectors property added by the with_functor_maps C++ mixin. - - This is basically a workaround for missing support for - Gaudi::Property> - - The optional `wrap` argument is an extra functor adapter that will be used - to wrap all of the functors in the dictionary. The canonical usage of this - feature is to add a wrapping `POD` functor to ensure plain, scalar data - types are returned. - """ - if input_dict is None: return {} - if wrap is not None: - input_dict = {k: wrap(v) for k, v in input_dict.items()} - return { - k: (v.code(), v.headers(), v.code_repr()) - for k, v in input_dict.items() - } diff --git a/Phys/FunctorCore/src/Cache.cpp b/Phys/FunctorCore/src/Cache.cpp index 591e4c1d55902f24be4f536d16473f7cb85ed9a4..01913f2143b6c1bf24aa902e6e65492fb9ba8cb1 100644 --- a/Phys/FunctorCore/src/Cache.cpp +++ b/Phys/FunctorCore/src/Cache.cpp @@ -25,9 +25,11 @@ namespace Functors::Cache { HashType makeHash( std::string_view data ) { return sv_hash( data ); } // Function used to generate the Gaudi component names from the hashes - std::string id( HashType hash ) { + std::string hashToStr( HashType hash ) { std::ostringstream oss; - oss << "functor_" << std::showbase << std::hex << hash; + oss << std::showbase << std::hex << hash; return oss.str(); } + std::string hashToFuncName( HashType hash ) { return "functor_" + hashToStr( hash ); } + } // namespace Functors::Cache diff --git a/Phys/FunctorCore/src/Components/ExampleAlg.cpp b/Phys/FunctorCore/src/Components/ExampleAlg.cpp index 07072d0b139c76ff4de12cd3c05d8bb3e72087a0..6f72783b50a50ff58d801c1687d84e582aaa7c3e 100644 --- a/Phys/FunctorCore/src/Components/ExampleAlg.cpp +++ b/Phys/FunctorCore/src/Components/ExampleAlg.cpp @@ -69,7 +69,7 @@ private: void decode() { m_factory.retrieve().ignore(); - m_pred = m_factory->get( this, m_functorproxy ); + m_factory->register_functor( this, m_pred, m_functorproxy ); } }; @@ -118,7 +118,7 @@ private: void decode() { m_factory.retrieve().ignore(); - m_pred = m_factory->get( this, m_functorproxy ); + m_factory->register_functor( this, m_pred, m_functorproxy ); } }; DECLARE_COMPONENT_WITH_ID( FunctorExampleAlg<>, "FunctorExampleAlg" ) diff --git a/Phys/FunctorCore/src/Components/Factory.cpp b/Phys/FunctorCore/src/Components/Factory.cpp index e0bb91de24f16ce3a507a51688c4498333fea206..f8e88e2b9b6dddac78330159c37211979e25f79f 100644 --- a/Phys/FunctorCore/src/Components/Factory.cpp +++ b/Phys/FunctorCore/src/Components/Factory.cpp @@ -12,351 +12,548 @@ #include "Functors/FunctorDesc.h" #include "Functors/IFactory.h" #include "GaudiKernel/Service.h" -#include "boost/algorithm/string/erase.hpp" -#include "boost/algorithm/string/predicate.hpp" -#include "boost/algorithm/string/replace.hpp" -#include "boost/algorithm/string/split.hpp" -#include "boost/algorithm/string/trim.hpp" -#include "boost/format.hpp" -#include "boost/lexical_cast.hpp" -#include +#include "fmt/format.h" +#include +#include +#include +#include +#include +#include #include #include #include /** @file Factory.cpp - * @brief Definitions of non-templated functor factory functions. + * @brief Implementation and documentation of the FunctorFactory + * + * The FunctorFactory is the service which enables our application to change + * functor expressions at configuration time. The following will try to give + * an overview of the different pieces that come together to enable the + * functionality of the FunctorFactory. + * + * A bit simplified, one can look at the entire functor framework as a system + * of 3 layers: + * 1. Functor configuration in python. + * 2. Translate the python objects to C++ Functor code. + * 3. Turn the C++ string representation into compiled and executable C++ + * + * We won't talk much more about the general functor implementation and level + * 1 and 2, for that please refer to the Moore documentation and Moore#284. + * The FunctorFactory only comes in at the 3rd level and is the service + * which is used by algorithms to turn a string representation into the + * final functor object. For that the algorithm registers a functor during + * the algorithm's initialize() step, like so: + * + * factory->register_functor( this, m_functor, m_functorproxy ); + * + * where m_functor will be filled with a functor object which is created from + * the content of the m_functorproxy property of the algorithm. + * + * An algorithms has to register all functors they plan to use within their + * initialize() method! + * + * Note that register_functor method does not instantly create/fill the + * `m_functor` it only registers the address of `m_functor` and will fill it + * in the `start()` call of ther service. (This is well before any algorithm + * will call its operator()) + * + * This postponed initialization of the functors in the service's start() call + * is a design decision that will become more obvious a bit later. + * + * When `start()` is called the FunctorFactory will do the following: (Note if + * you've already heard of the FunctorCache, this will be explained at the + * end, the following will first focus on the just in time compilation + * workflow) + * + * 1. Write the C++ representation of functors into a configurable amount + * (property: m_split) of temporary c++ files (see write_cpp_files + * function). The actual code that we will compile for each functor is + * generated in the do_register function. As you can see we wrap the actual + * c++ functor code in a bit of Gaudi::PluginService magic, and end up + * defining one variable per functor of the following form: + * + * Gaudi::PluginService::DeclareFactory> + * functor_0x50805fa6b888b0a0{ + * std::string{"0x50805fa6b888b0a0"}, + * []() -> std::unique_ptr { + * return std::make_unique>(::Functors::AcceptAll{});; + * } + * }; + * + * This is done to retrieve the functors in an easy way via the + * PluginService, see point 3 below. The first argument to the constructor + * is a hash that identifies this functor and is calculated based on the + * code of the functor(see do_register()). The second argument is a lambda + * which when called will return a unique_ptr to the newly created functor + * object that we are aiming to create. In Gaudi::PlugingService lingo, this + * is the factory function to create the registered component. + * + * 2. Invoke the functor_jitter python script (generated in + * FunctorCore/CMakeLists.txt) which will compile the c++ files into a + * shared library while using a configurable (property: m_jit_n_jobs) + * amount of threads. The compiler used by the functor_jitter is the same + * compiler + same sets of flags that were used to compile the Rec project + * during the build step. Compared to previous solutions that were using + * ROOT's cling to compile functors, this has the benefit that the code is + * properly optimized, vectorized, and overall indistinguishable from code + * that would have been built during the build step (e.g. the + * FunctorCache). If you look at the created cpp files from step 1. you + * will notice that they do not have any include statements. Instead the + * functor_jitter script invokes the compiler with the -include flag which + * will include a preprocessed header that was created during the build + * step. Check FunctorCore/CMakeLists.txt for more details on how this + * header is created. It is important that this header has sufficient + * includes to be able to correctly compile any functor that could be + * specified at configuration time. To have this preprocessed header was a + * design decision that enables us to make the JIT compilation during + * runtime independent of the system that we run on and its available + * system headers. + * + * 3. First we open (dlopen) the shared library we just created. This is where + * the previously mentioned Gaudi::PluginService magic comes into play. The + * opening of the shared library will call the constructors of the defined + * variables, which in turns registers the functors with the global + * Gaudi::PluginService. Using the hash of a functor which functions as a + * key to the PluginSetvice, we are at this point able to ask the + * PluginService to simply create the various functor objects we need and + * then bind them to the algorithms. + * (This is done at the end of the start() function. If you are wondering + * what do_copy does please have a look at the do_copy lambda in + * IFactory.h) + * + * + * Note, the location where the JIT shared library is created is configurable + * (property: m_jit_lib_dir) . Before the FunctorFactory starts steps 1-3 from + * above, it does check if the library already exists in that location, thus + * making reuse of this library possible across jobs. This only works for + * exact matches though as the library name is defined as a hash of the + * dependencies (functor_jitter+preprocessed header) and the content of the + * cpp files. + * + * + * The above explains how the FunctorFactory works in JIT mode, but we can + * also create a FunctorCache. This is a way to pre-compile the functors used + * in a certain configuration during the Project's build step. For an example, + * look at Rec/Phys/FunctorCache/CMakeLists.txt Given a certain options file + * we want to create a cache for, the procedure is similar to the older LoKi + * style functor caches. During the build step we start a gaudi job with + * specific options for the FunctorFactory such that we only create the cpp + * files and don't start any JIT compilation. + * + * The generated cpp files are expected by cmake which takes care of + * configuring and building a shared library based on those files which we + * commonly only refer to as functorcache. + * + * If using the cache is enabled, the do_register function will first check + * via the Gaudi::PluginService if the required functor is already in a + * functorcache and only registers the functor for JIT if it couldn't be + * found. This first lookup only considers proper functor caches configured + * via cmake, not previously built libraries via JIT + * + * + * Note on data dependencies: + * A functor can have data dependencies by inheriting from DataDepWrapper + * (Functors/Utilities.h) When a functor is created and then bound to an + * algorithm, it will automatically create the required DataHandles which it + * will need to have access to the TES objects in its operator(). This + * creation automatically registers the DataHandles with the algorithm that + * owns the functor. + * + * The Problem with the above procedure is that, the creation and bind of + * functors and thus the registration of DataHandles doesn't happen until the + * `start()` call of the FunctorFactory. But data dependencies are resolved + * and checked by the HiveDataBroker during its initialize step. So at the + * time the HiveDataBroker interrogates an algorithm to determine its required + * inputs and outputs, the DataHandles of functors haven't yet been registered + * with the algorithm. + * + * The solution to this problem has two steps: + * 1. During configuration (PyConf), the data dependencies of functors are + * recognized and used to populate the owning algorithm's ExtraInputs + * property. This means the HiveDataBroker will get the correct "reply" + * when checking an algorithm's inputs and outputs. + * 2. During the creation of a Functor's DataHandles, we check the owning + * algorithms ExtraInputs property to assert that this property already + * contains the TES location we are currently creating a DataHandle for. + * This check should be sufficient to catch any misconfiguration and throw + * meaningful error messages. + * */ namespace { - std::ostream& include( std::ostream& os, std::string_view header ) { - if ( header.empty() ) { throw GaudiException{"Got empty header name", "FunctorFactory", StatusCode::FAILURE}; } - os << "#include "; - if ( header.front() == '<' && header.back() == '>' ) { - os << header; - } else { - os << '"' << header << '"'; - } - return os << '\n'; - } - - std::string make_default_cppname( std::string cppname ) { - if ( boost::algorithm::starts_with( cppname, "ToolSvc." ) ) { cppname.erase( 0, 8 ); } - std::replace_if( - cppname.begin(), cppname.end(), []( char c ) { return c == '.' || c == ' '; }, '_' ); - boost::algorithm::replace_all( cppname, "::", "__" ); - cppname.insert( 0, "FUNCTORS_" ); - return cppname; - } - std::unique_ptr openFile( std::string namebase, unsigned short findex, - std::vector const& lines, - std::vector const& headers ) { - // construct the file name - // 1) remove trailing .cpp - if ( boost::algorithm::ends_with( namebase, ".cpp" ) ) boost::algorithm::erase_tail( namebase, 4 ); - // 2) replace blanks by underscore - std::replace( namebase.begin(), namebase.end(), ' ', '_' ); - // 3) construct the name - boost::format fname( "%s_%04d.cpp" ); - fname % namebase % findex; - auto file = std::make_unique( fname.str() ); - // write the include directives - *file << "// Generic statements\n"; - for ( const auto& l : lines ) { *file << l << '\n'; } - *file << "// Explicitly declared include files:\n"; - for ( const auto& h : headers ) { include( *file, h ); } - return file; - } - - /** Write out the entry in a generated functor cache .cpp file for a single functor. + /** + * @brief helper function for LibHandle + * + * @param handle pointer to library on which we call dlclose */ - std::ostream& makeCode( std::ostream& stream, Functors::Cache::HashType hash, std::string_view type, - std::string_view code ) { - boost::format declareFactory{R"_(namespace { - using t_%1% = %2%; - Gaudi::PluginService::DeclareFactory functor_%1%{ - Functors::Cache::id( %1% ), - []() -> std::unique_ptr { - return std::make_unique( - %3% - ); - } + void lib_closer( void* handle ) { + if ( handle != nullptr ) { dlclose( handle ); } }; -} -)_"}; - return stream << ( declareFactory % boost::io::group( std::showbase, std::hex, hash ) % type % code ); - } + /** + * unique_ptr specialization to hold the void* returned by dlopen which will + * take care of calling dlclose() on the library upon destruction. Most + * important for the scenario where we potentially reinitialize many times + * and thus maybe create and open many functor libraries. + */ + using LibHandle = std::unique_ptr; } // namespace +// support for Gaudi::Property which can not be +// implemented yet in Gaudi because cling will crash due to +// https://github.com/root-project/root/issues/9670 +namespace Gaudi::Parsers { + StatusCode parse( std::filesystem::path& result, const std::string& input ) { + result = std::filesystem::path{input}; + return StatusCode::SUCCESS; + } +} // namespace Gaudi::Parsers + /** @class FunctorFactory * * This service does all the heavy lifting behind compiling functors into - * type-erased Functor objects. It can do this either via ROOT's - * just-in-time compilation backend, or by using a pre-compiled functor - * cache. The tool is also responsible for generating the code that - * produces the precompiled functor cache. It is heavily inspired by Vanya - * Belyaev's LoKi::Hybrid::Base. + * type-erased Functor objects. It can do this either via on demand + * invoking the same compiler the project was compiled with, or by using a + * pre-compiled functor cache. */ struct FunctorFactory : public extends { using extends::extends; - // pointer-to-function type we use when JIT compiling - using factory_function_ptr = functor_base_t ( * )(); - - functor_base_t get_impl( Gaudi::Algorithm* owner, std::string_view functor_type, ThOr::FunctorDesc const& desc, - CompilationBehaviour compile ) override { - // Combine the 'compile' argument with the global settings to determine - // what compilation methods we should try - bool const fail_hard = compile & ExceptionOnFailure; - bool const use_cling{this->m_use_cling && ( compile & TryJIT )}; - bool const use_cache{this->m_use_cache && ( compile & TryCache )}; - - // Prepare the string that fully specifies the functor we want to retrieve -- basically the combination of - // input type, output type, functor string - // First, sort and de-duplicate the headers - auto headers = desc.headers; - std::sort( headers.begin(), headers.end() ); - headers.erase( std::unique( headers.begin(), headers.end() ), headers.end() ); - if ( msgLevel( MSG::DEBUG ) ) { - debug() << "Decoding " << desc.code << endmsg; - debug() << "With extra headers:"; - for ( auto const& header_line : headers ) { debug() << " " << header_line; } - debug() << endmsg; + void do_register( std::function )> do_copy, std::string_view functor_type, + ThOr::FunctorDesc const& desc ) override { + // If we are already in RUNNING state we won't go through start() thus a + // do_register call doesn't make sense as the library won't be compiled and + // the Functor won't get resolved. See GaudiKernel/StateMachine.h for info + // on possible states and transitions. + if ( FSMState() == Gaudi::StateMachine::RUNNING ) { + throw GaudiException{"do_register needs to be called before start()", "FunctorFactory", StatusCode::FAILURE}; } - // FIXME it seems that having a quoted string in the middle of the - // string adds quotes at the start/end...need Gaudi!919 - std::size_t findex{desc.code.front() == '"'}, codelen{desc.code.size() - findex - ( desc.code.back() == '"' )}; - auto trimmed_code = std::string_view{desc.code}.substr( findex, codelen ); - - // This is basically Functor( PTCUT( ... ) ... ) - std::string full_code{functor_type}; - full_code.append( "( " ); - full_code.append( trimmed_code ); - full_code.append( " )" ); - + // This is basically std::make_unique>( PTCUT( ... ) ... ) + auto func_body = "return std::make_unique<" + std::string( functor_type ) + ">(" + std::string( desc.code ) + ");"; // Now we can calculate the hash - const auto hash = Functors::Cache::makeHash( full_code ); - - if ( msgLevel( MSG::VERBOSE ) ) { - verbose() << "Full string for hash: " << full_code << endmsg; - verbose() << "Resulting hash is " << std::hex << std::showbase << hash << endmsg; + const auto hash = Functors::Cache::makeHash( func_body ); + + // Check via the PluginService if the functor is already present in a + // prebuilt functor cache, if not disabled. + + if ( !m_disable_cache ) { + auto cached_functor = + ::Gaudi::PluginService::Factory::create( Functors::Cache::hashToStr( hash ) ); + if ( cached_functor ) { + do_copy( std::move( cached_functor ) ); + if ( msgLevel( MSG::VERBOSE ) ) { + verbose() << "Functor: " << desc.repr << " with hash: " << Functors::Cache::hashToStr( hash ) + << " found in cache." << endmsg; + } + // becasue we don't JIT this functor, the FunctorFactory doesn't need to + // keep track of any info for this functor. We have already resolved it + // completely so we can return early + return; + } } - // The object we'll eventually return - functor_base_t functor; - - // See if we can magically load the functor from the cache (Gaudi magic!) - // Don't bother trying if we were told not to - if ( use_cache ) { - functor = ::Gaudi::PluginService::Factory::create( Functors::Cache::id( hash ) ); - if ( functor ) { - functor->setJITCompiled( false ); - } else if ( !functor && !use_cling ) { - // We print a different INFO message below if use_cling is true - info() << "Cache miss for functor: " << trimmed_code << endmsg; - } + if ( msgLevel( MSG::VERBOSE ) ) { + verbose() << "Functor: " << desc.repr << " with hash: " << Functors::Cache::hashToStr( hash ) + << " registered for JIT." << endmsg; } - // Shorthand for throwing an informative exception - auto exception = [functor_type]( auto const& name ) { - std::string ourname{"FunctorFactory::get<"}; - ourname.append( functor_type ); - ourname.append( ">" ); - return GaudiException{name, std::move( ourname ), StatusCode::FAILURE}; - }; + // See if we already JIT compiled this functor and can therefore reuse + // the factory function that we got before + auto iter = m_all.find( hash ); + if ( iter != m_all.end() ) { + if ( msgLevel( MSG::VERBOSE ) ) { verbose() << "Found already registered functor: " << desc.repr << endmsg; } + iter->second.first.push_back( do_copy ); + } else { - if ( !functor && use_cling ) { - // See if we already JIT compiled this functor and can therefore reuse - // the factory function that we got before - auto iter = m_factories.find( hash ); - if ( iter != m_factories.end() ) { - // We had already JIT compiled this functor - info() << "Reusing cling compiled factory for functor: " << trimmed_code << endmsg; - auto factory_function = iter->second; - functor = factory_function(); - functor->setJITCompiled( true ); - } else { - // Need to actually do the JIT compilation - if ( use_cache ) { - info() << "Cache miss for functor: " << trimmed_code << ", now trying cling with headers " << headers - << endmsg; - } else { - info() << "Using cling for functor: " << trimmed_code << " with headers " << headers << endmsg; - } + // fmt::format turns {{ -> { + std::string code_format = R"_( +::Gaudi::PluginService::DeclareFactory> +functor_{0}{{ + std::string{{"{0}"}}, + []() -> std::unique_ptr {{ + {1}; + }} +}}; +)_"; - // The expression we ask cling to compile is not quite the same as - // 'full_code'. Instead of Functor( PT > ... ) we ask it to - // compile the declaration of a function returning functor_base_t that - // takes no arguments. We then ask cling to give us the address of - // this function and call it ourselves. This looks like: - // functor_base_t functor_0xdeadbeef() { - // return std::make_unique>( PT > ... ); - // } - std::ostringstream code; - - // Enable -O3-like code optimisation - code << "#pragma cling optimize(3)\n"; - - // Work around cling not auto-loading based on functions...? - code << "#pragma cling load(\"TrackKernel\")\n"; - - // Workaround for cling errors along the lines of: - // unimplemented pure virtual method 'i_cast' in 'RelationWeighted' - // and several others. See https://gitlab.cern.ch/gaudi/Gaudi/issues/85 - // for discussion of a better solution. -#ifdef GAUDI_V20_COMPAT - code << "#ifndef GAUDI_V20_COMPAT\n"; - code << "#define GAUDI_V20_COMPAT\n"; - code << "#endif\n"; -#endif -#ifdef USE_DD4HEP - code << "#define USE_DD4HEP\n"; -#endif - - // Include the required headers - for ( auto const& header : headers ) { include( code, header ); } - - // Get the name for the factory function. Add a suffix to avoid it - // matching the cache entries. - auto function_name = Functors::Cache::id( hash ) + "_cling"; - - // Declare the factory function - code << functor_base_t_str << " " << function_name << "() { return std::make_unique<" << functor_type << ">( " - << trimmed_code << " ); }\n"; - - // Assign its address to a variable to trigger compilation while the pragmas are active - code << "auto const " << function_name << "_factory_ptr = " << function_name + ";"; - - if ( msgLevel( MSG::VERBOSE ) ) { verbose() << "Full code to JIT is:\n" << code.str() << endmsg; } - - // Try and JIT compile this expression - if ( !gInterpreter->Declare( code.str().c_str() ) ) { - info() << "Code we attempted to JIT compile was:\n" << code.str() << endmsg; - throw exception( "Failed to JIT compile functor" ); - } + auto hash_str = Functors::Cache::hashToStr( hash ); + auto function_name = Functors::Cache::hashToFuncName( hash ); - // Get the address of the factory function we just JITted - TInterpreter::EErrorCode error_code{TInterpreter::kNoError}; - auto value = gInterpreter->Calc( ( function_name + "_factory_ptr" ).c_str(), &error_code ); - if ( error_code != TInterpreter::kNoError ) { - throw exception( "Failed to get the factory function address, error code " + std::to_string( error_code ) ); - } + auto cpp_code = fmt::format( code_format, hash_str, func_body ); - // Cast the function pointer to the correct type - auto factory_function = reinterpret_cast( value ); + m_all.emplace( hash, std::pair{std::vector{do_copy}, cpp_code} ); + } + } - // Save the factory function pointer - m_factories.emplace( hash, factory_function ); + /** + * @brief split functors in m_all into N cpp files and write those. + * + * Split strategy is controled by the value of m_split. + * m_split < 0 -> split such that each file contains abs(m_split) + * m_split > 0 -> split into m_split files, always creates m_split files even if fewer functors are registered. + * m_split == 0 -> is mapped to 1 in the property update handler + * + * If splitting leads to uneven division, remaining functors are spread over the existing files. + * Used pattern to name files: "FUNCTORS_FunctorFactory_{:04d}.cpp"" + * + * FIXME what negative value is the sweet spot for JIT? + * + * @param dest_dir Destination directory files are written to. + */ + void write_cpp_files( std::filesystem::path const& dest_dir ) const { + auto const cpp_filename = dest_dir / "FUNCTORS_FunctorFactory_{:04d}.cpp"; + + auto split = m_split > 0 ? m_split.value() : m_all.size() / std::abs( m_split ); + + std::vector files( split ); + for ( std::size_t i{0}; i < split; ++i ) { files[i] = fmt::format( cpp_filename.string(), i + 1 ); } + + // integer division will give us the amount of functors per file. + auto const functors_per_file = m_all.size() / split; + // the remainder tells us how many files will get 1 extra functor. + auto const remainder = m_all.size() % split; + // NOTE if we have less functors than required by split (m_all.size() < + // split) we still write all N=split files as the functor cache code in + // cmake expects these files to be created + + auto open_and_check = [disabe_jit = m_disable_jit]( std::ofstream& out, std::filesystem::path const& fname ) { + out.open( fname ); + if ( !out.is_open() ) { + throw GaudiException{"Failed to open file " + fname, "FunctorFactory", StatusCode::FAILURE}; + } + // if jit is disabled we assume this is for the functor cache so we + // include the unprocessed header + out << ( disabe_jit ? "#include \"Functors/JIT_includes.h\"\nnamespace {\n" : "namespace {\n" ); + }; - // Use the JITted factory function - functor = factory_function(); - functor->setJITCompiled( true ); + // NOTE + // m_all is a map indexed with the hash, which guarantees that the + // iteration order is stable and does not depend on the order of + // `register_functor` calls. + auto functors_iter = m_all.begin(); + auto out = std::ofstream{}; + for ( std::size_t file_idx{0}; file_idx < files.size(); ++file_idx ) { + open_and_check( out, files[file_idx] ); + std::size_t written_functors{0}; + while ( functors_iter != m_all.end() && ++written_functors <= ( functors_per_file + ( file_idx < remainder ) ) ) { + out << functors_iter++->second.second; } + out << "\n}"; + out.close(); } + } - if ( functor ) { - functor->bind( owner ); - } else if ( fail_hard && ( use_cache || use_cling ) ) { - // Don't emit too many messages while generating the functor caches. In - // that case both cling and the cache are disabled, so we will never - // actually retrieve a functor here. - std::string error_message{"Couldn't load functor using ["}; - if ( use_cache && use_cling ) { - error_message += "cache, cling"; - } else if ( use_cache ) { - error_message += "cache"; - } else { - error_message += "cling"; + StatusCode initialize() override { + auto sc = Service::initialize(); + + // we will fill this buffer with the content of functor_jitter and the + // preprocessed header to then calcuate a hash and write that value into + // m_header_hash which will be used to define a JIT library's filename + std::stringstream buffer; + + // first we need to find the "functor_jitter" script on the runtime_path. + // That env var looks like "path1:path2:path3...." + auto const runtime_path = System::getEnv( "PATH" ); + auto begin = 0U; + auto end = runtime_path.find( ':' ); + auto found_functor_jitter = std::filesystem::path{}; + + while ( end != std::string::npos ) { + auto path = std::filesystem::path{runtime_path.substr( begin, end - begin )}; + path.append( "functor_jitter" ); + if ( std::filesystem::is_regular_file( path ) ) { + found_functor_jitter = path; + break; } - error_message += "]: " + desc.repr; - throw exception( std::move( error_message ) ); + // move new begin to just after the ":" + begin = end + 1; + end = runtime_path.find( ':', begin ); + } + // if we didn't find anything and break out of the while loop, we need to + // check the last element + if ( end == std::string::npos ) { + auto path = std::filesystem::path{runtime_path.substr( begin, end - begin )}; + path.append( "/functor_jitter" ); + if ( std::filesystem::is_regular_file( path ) ) { found_functor_jitter = path; } } - // If we're going to write out the .cpp files for creating the functor cache when we finalise then we need to - // store the relevant data in an internal structure - if ( this->m_makeCpp ) { - // Store the functor alongside others with the same headers - m_functors[std::move( headers )].emplace( hash, functor_type, trimmed_code ); + if ( found_functor_jitter.empty() ) { + error() << "Could not find 'functor_jitter' executable on runtime path!" << endmsg; + return StatusCode::FAILURE; + } + debug() << "Calculating hash of: " << found_functor_jitter << endmsg; + buffer << std::ifstream{found_functor_jitter}.rdbuf(); + + // runtime environment variable which points to the preprocessed header + auto const path_to_header = System::getEnv( "FUNCTORFACTORY_PREPROCESSED_HEADER" ); + if ( path_to_header == "UNKNOWN" ) { + error() << "Could not retrieve path to preprocessed header from env var: FUNCTORFACTORY_PREPROCESSED_HEADER" + << endmsg; + return StatusCode::FAILURE; } - return functor; + buffer << std::ifstream{path_to_header}.rdbuf(); + m_dependencies_hash = Functors::Cache::hashToStr( Functors::Cache::makeHash( buffer.str() ) ); + debug() << "Calculating hash of: " << path_to_header << endmsg; + debug() << "FunctorFactory initialized with dependencies hash: " << m_dependencies_hash << endmsg; + return sc; } - /** Write out the C++ files needed to compile the functor cache if needed. - */ - StatusCode finalize() override { - if ( m_makeCpp ) { writeCpp(); } - return Service::finalize(); - } + StatusCode start() override { + auto sc = Service::start(); + if ( m_all.empty() ) { return sc; } + + if ( msgLevel( MSG::DEBUG ) ) { debug() << m_all.size() << " functors were registered" << endmsg; } + + // FIXME Old FunctorFactory had this namespace but do I need it? + std::string full_lib_code{"namespace {\n"}; + // m_all is a map indexed with the hash, which guarantees that the + // iteration order is stable and does not depend on the order of + // `register_functor` calls. + for ( auto const& entry : m_all ) { full_lib_code += entry.second.second; } + full_lib_code += "}"; + + auto const content_hash = Functors::Cache::hashToStr( Functors::Cache::makeHash( full_lib_code ) ); + auto const file_prefix = "FunctorJitLib_" + m_dependencies_hash + "_" + content_hash; + auto const lib_filename = file_prefix + ".so"; + auto const lib_full_path = m_jit_lib_dir / lib_filename; + // declare the variable in outer scope to be able to cleanup at the end. + std::filesystem::path tmp_dir; + + // We first check if we have already compiled this library in a previous + // execution of the same job by looking for full path to the lib inside the + // FunctorJitLibDir (e.g. /tmp/libname.so) + if ( m_lib_handle = LibHandle{dlopen( lib_full_path.c_str(), RTLD_LOCAL | RTLD_LAZY ), lib_closer}; + m_lib_handle != nullptr ) { + info() << "Reusing functor library: " << lib_full_path << endmsg; + } else { + + // In a multi job scenario we want to avoid many jobs directly writing to + // the same file and potentially causing problems. Thus, in JIT mode we + // will create all files in a subdirectory that is unique for each + // process and only at the end move the created library one directory up + // into m_jit_lib_dir. If two jobs create the same lib and try to move at + // the same time we are still save because the rename/move operation is + // atomic in the sense that a dlopen call will either load the already + // existing file or the newly renamed one. But it can't fail due to e.g. + // the file not having been fully overwritten or similar weird things. + // + // But if m_disable_jit is set we are + // running just for the purpose of creating the cpp files so we dump them + // directly in m_jit_lib_dir + tmp_dir = m_jit_lib_dir; + if ( !m_disable_jit ) { + tmp_dir.append( file_prefix + "_" + std::to_string( getpid() ) ); + std::filesystem::remove_all( tmp_dir ); + std::filesystem::create_directory( tmp_dir ); + } -protected: - using HashType = Functors::Cache::HashType; - using FactoryCache = std::map; - using FunctorSet = std::map, std::set>>; - FunctorSet m_functors; // {headers: [{hash, type, code}, ...]} - FactoryCache m_factories; // {hash: factory function pointer, ...} + write_cpp_files( tmp_dir ); - /** @brief Generate the functor cache .cpp files. - * - * In order to expose as many bugs as possible, make sure that we generate a - * different .cpp file for every set of requested headers, so every functor - * is compiled with *exactly* the headers that were requested. - */ - void writeCpp() const { - /** The LoKi meaning of this parameter was: - * - positive: write N-files - * - negative: write N-functors per file - * - zero : write one file - * currently it is not fully supported, we just use it to check that CMake - * is aware of at least as many source files as the minimum we need. - * - * @todo When split > m_functors.size() then split the functors across - * more files until we are writing to all 'split' available source - * files. - */ - std::size_t split{0}; - if ( !boost::conversion::try_lexical_convert( System::getEnv( "LOKI_GENERATE_CPPCODE" ), split ) ) { split = 0; } - if ( m_functors.size() > split ) { - throw GaudiException{"Functor factory needs to generate at least " + std::to_string( m_functors.size() ) + - " source files, but LOKI_GENERATE_CPPCODE was set to " + std::to_string( split ) + - ". Increase the SPLIT setting in the call to loki_functors_cache() to at least " + - std::to_string( m_functors.size() ), - "FunctorFactory", StatusCode::FAILURE}; - } + // This branch is used for the FunctorCache creation, where we only want + // to write the files but not JIT them. + if ( m_disable_jit ) { + warning() + << "Current configuration requires new functor library but property DisableJIT is enabled! Some functors " + "will not be initialized!" + << endmsg; + return sc; + } - /** We write one file for each unique set of headers - */ - unsigned short ifile{0}; - for ( auto const& [headers, func_set] : m_functors ) { - std::unique_ptr file; - unsigned short iwrite{0}; - for ( auto const& [hash, functor_type, brief_code] : func_set ) { - if ( !file ) { file = openFile( m_cppname, ++ifile, m_cpplines, headers ); } + info() << "New functor library will be created." << endmsg; + if ( msgLevel( MSG::DEBUG ) ) { debug() << "Based on generated C++ files in folder: " << tmp_dir << endmsg; } - *file << '\n' << std::dec << std::noshowbase << "// FUNCTOR #" << ++iwrite << "/" << func_set.size() << '\n'; + // functor_jitter is a shell script generated by cmake to invoke the + // correct compiler with the correct flags see: + // Phys/FunctorCore/CMakeLists.txt + auto cmd = "functor_jitter " + std::to_string( m_jit_n_jobs ) + " " + tmp_dir + " " + lib_filename; - // write actual C++ code - ::makeCode( *file, hash, functor_type, brief_code ); + if ( msgLevel( MSG::VERBOSE ) ) { verbose() << "Command that will be executed:\n" << cmd << endmsg; } - *file << '\n'; + auto start_time = std::chrono::high_resolution_clock::now(); + auto return_code = std::system( cmd.c_str() ); + auto total_time = + std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - start_time ); + info() << "Compilation of functor library took " << total_time.count() << " seconds" << endmsg; + if ( return_code != 0 ) { throw GaudiException{"Non zero return code!", "FunctorFactory", StatusCode::FAILURE}; } + + auto const lib_tmp_full_path = tmp_dir / lib_filename; + if ( msgLevel( MSG::VERBOSE ) ) { + verbose() << "Rename " << lib_tmp_full_path << " to " << lib_full_path << endmsg; } + std::filesystem::rename( lib_tmp_full_path, lib_full_path ); + + m_lib_handle = LibHandle{dlopen( lib_full_path.c_str(), RTLD_LOCAL | RTLD_LAZY ), lib_closer}; } - // Make sure the remaining files are empty. This ensures generated code - // from previous builds (with more functors) is overwritten and does not - // interfere with the new build. - while ( ifile < split ) openFile( m_cppname, ++ifile, {}, {} ); - } + if ( m_lib_handle == nullptr ) { + throw GaudiException{std::string( "dlopen Error:\n" ) + dlerror(), "FunctorFactory", StatusCode::FAILURE}; + } + + // at this point we have compiled the functors so now it is time to make + // sure that we initialize each algorithm's functors + for ( auto const& entry : m_all ) { + for ( auto const& do_copy : entry.second.first ) { + // registration of the functors with the PluginService happens + // automatically when the library is opened, thus we can now simply go + // via the PluginService to get our functors + auto functor = ::Gaudi::PluginService::Factory::create( + Functors::Cache::hashToStr( entry.first ) ); + do_copy( std::move( functor ) ); + } + } - // Flags to steer the use of cling and the functor cache - Gaudi::Property m_use_cache{this, "UseCache", System::getEnv( "LOKI_DISABLE_CACHE" ) == "UNKNOWN"}; - Gaudi::Property m_use_cling{this, "UseCling", System::getEnv( "LOKI_DISABLE_CLING" ) == "UNKNOWN"}; - Gaudi::Property m_makeCpp{this, "MakeCpp", System::getEnv( "LOKI_GENERATE_CPPCODE" ) != "UNKNOWN"}; + // if we reach this point then everything probably went well. Thus if we + // aren't running in DEBUG or more verbose, let's cleanup our temporary + // files. if tmp_dir is empty we just loaded an exisiting lib and don't + // need to cleanup. m_disable_jit also shouldn't do a cleanup but that one + // does an early retun so we never get to this line here + if ( msgLevel() > MSG::DEBUG && !tmp_dir.empty() ) { std::filesystem::remove_all( tmp_dir ); } + return sc; + } - // Properties steering the generated functor cache code - Gaudi::Property m_cppname{this, "CppFileName", make_default_cppname( this->name() )}; - Gaudi::Property> m_cpplines{this, "CppLines", {"#include \"Functors/Cache.h\""}}; +private: + std::map )>>, std::string>> + m_all; + + Gaudi::Property m_disable_jit{this, "DisableJIT", System::getEnv( "THOR_DISABLE_JIT" ) != "UNKNOWN"}; + Gaudi::Property m_disable_cache{this, "DisableCache", System::getEnv( "THOR_DISABLE_CACHE" ) != "UNKNOWN"}; + Gaudi::Property m_jit_lib_dir{this, "JitLibDir", System::getEnv( "THOR_JIT_LIBDIR" ), + [this]( auto& /*unused*/ ) { + m_jit_lib_dir = ( ( m_jit_lib_dir.value() == "UNKNOWN" ) + ? std::filesystem::temp_directory_path() + : m_jit_lib_dir.value() ); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}; + + // a value of -1 means we will use as all threads available. Note that this + // property is connected to m_split. Thus, the actual amount of threads + // started are m_split up a maxiumum of to m_jit_n_jobs threads. + Gaudi::Property m_jit_n_jobs{ + this, "JITNJobs", 1, + [this]( auto& /*unused*/ ) { + if ( int tmp{}; boost::conversion::try_lexical_convert( System::getEnv( "THOR_JIT_N_JOBS" ), tmp ) ) { + m_jit_n_jobs = tmp; + } + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}; + // meaning of m_split is explained in write_cpp_files() + Gaudi::Property m_split{ + this, "Split", 1, + [this]( auto& /*unused*/ ) { + if ( int tmp{}; boost::conversion::try_lexical_convert( System::getEnv( "LOKI_GENERATE_CPPCODE" ), tmp ) ) { + m_split = tmp; + } + // map 0 -> 1 + m_split = m_split == 0 ? 1 : m_split.value(); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}; + + // Gaudi::Property tmp{this, "tmp", std::filesystem::path{"ab/cd"}}; + std::string m_dependencies_hash{}; + LibHandle m_lib_handle{nullptr, lib_closer}; }; DECLARE_COMPONENT( FunctorFactory ) diff --git a/Phys/FunctorCore/src/FunctorDesc.cpp b/Phys/FunctorCore/src/FunctorDesc.cpp index 5fecd6378628b024b80e98384908409bd844868d..39966a082d345ad87a91a02ba8dafd0f4ed118d7 100644 --- a/Phys/FunctorCore/src/FunctorDesc.cpp +++ b/Phys/FunctorCore/src/FunctorDesc.cpp @@ -11,11 +11,26 @@ #include "Functors/FunctorDesc.h" namespace std { + /** + * @brief operator<< specialization for a Functor Description (FuntorDesc) + * + * Output should match the python repr result, e.g for the PT Functor: + * "('::Functors::Track::TransverseMomentum{}', ['Functors/TrackLike.h'], 'PT')" + * + * @param o stream to output into + * @param f FunctorDesc to stream into o + * @return ostream& filled with the string representation of f + */ std::ostream& operator<<( std::ostream& o, ThOr::FunctorDesc const& f ) { - return GaudiUtils::details::ostream_joiner( o << "\"(" << std::quoted( f.code, '\'' ) << ", " - << "['", - f.headers, "', '" ) - << "']" - << ", " << std::quoted( f.repr, '\'' ) << ")\""; + // we can't use the default operator<< for the std::vector of headers + // because we need the single quotes around the header to match the python + // repr output of a Funtor see above + o << "\"(" << std::quoted( f.code, '\'' ) << ", ["; + if ( !f.headers.empty() ) { + // this if is to avoid having [''] instead of [] if f.headers is empty + GaudiUtils::details::ostream_joiner( o << "'", f.headers, "', '" ) << "'"; + } + o << "], " << std::quoted( f.repr, '\'' ) << ")\""; + return o; } } // namespace std diff --git a/Phys/FunctorCore/src/functor_jit_dummy/test_includes.cpp b/Phys/FunctorCore/src/functor_jit_dummy/test_includes.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bb61df9677abfb6173a21d0aba4979f77acdf558 --- /dev/null +++ b/Phys/FunctorCore/src/functor_jit_dummy/test_includes.cpp @@ -0,0 +1,13 @@ +/*****************************************************************************\ +* (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration * +* * +* This software is distributed under the terms of the GNU General Public * +* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". * +* * +* In applying this licence, CERN does not waive the privileges and immunities * +* granted to it by virtue of its status as an Intergovernmental Organization * +* or submit itself to any jurisdiction. * +\*****************************************************************************/ +#include "Functors/JIT_includes.h" + +int main() { return 0; } diff --git a/Phys/FunctorCore/tests/options/functor_datahandle_test.py b/Phys/FunctorCore/tests/options/functor_datahandle_test.py new file mode 100644 index 0000000000000000000000000000000000000000..99ae757ee2c4f5fb282f93960ff9089a7954b9de --- /dev/null +++ b/Phys/FunctorCore/tests/options/functor_datahandle_test.py @@ -0,0 +1,52 @@ +############################################################################### +# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +import os +from Gaudi.Configuration import ApplicationMgr, VERBOSE +from Configurables import Gaudi__Monitoring__MessageSvcSink as MessageSvcSink +from Configurables import EvtStoreSvc +from Functors import SIZE + +# algorithms are coming from PyConf because we need to use DataHandles etc. +from PyConf.Algorithms import FunctorExampleAlg as FEA, Gaudi__Examples__VectorDataProducer as VDP +from PyConf.dataflow import dataflow_config + +# user verbose so we can see the DH registration in the output +app = ApplicationMgr(OutputLevel=VERBOSE) +# FEA has counters so we need a sink +app.ExtSvc.append(MessageSvcSink()) +# why does the default EventDataSvc not work? good question -> Gaudi#218 +whiteboard = EvtStoreSvc("EventDataSvc", EventSlots=1) +app.ExtSvc.append(whiteboard) + +vdp = VDP(name="VDP") + +# this env var is set from inside the test_functor_string_datahandle.qmt file. +# For that test we pass a string instead of the datahandle into the functor, +# which works in pyconf as long as we allow strings as DataHandles... +# But this behaviour is not supported for functors and we check via this test +# that this is correctly caught on the C++ side and throws: +# ERROR TES::Size Usage of DataHandle["/Event/VDP/OutputLocation"] +# in Functor (TES::Size) requires that owning algorithm FEA contains this TES +# location inside the ExtraInputs property. This is likely a Configuration/PyConf bug! +if os.environ.get("TEST_FUNCTORS_DH_USE_STRING", False): + fea = FEA(name="FEA", Cut=SIZE(vdp.OutputLocation.location) < 5) +else: + fea = FEA(name="FEA", Cut=SIZE(vdp.OutputLocation) < 5) + +c = dataflow_config() +c.update(fea.configuration()) +algs, _ = c.apply() +app.TopAlg = algs + +# - Event +app.EvtMax = 1 +app.EvtSel = "NONE" +app.HistogramPersistency = "NONE" diff --git a/Phys/FunctorCore/tests/options/test_functors.py b/Phys/FunctorCore/tests/options/test_functors.py index a53db6ed008f95cc633ede461eabd6e5e847e244..37d54ba63725852bc3b3848d35cf95ef94f30bc1 100644 --- a/Phys/FunctorCore/tests/options/test_functors.py +++ b/Phys/FunctorCore/tests/options/test_functors.py @@ -14,11 +14,10 @@ # @author Saverio Mariani ## # ============================================================================= -import Configurables +from PyConf import Algorithms from Configurables import (ApplicationMgr, LHCbApp) from Functors import * from Functors.tests.categories import DUMMY_DATA_DEP -from Functors.utils import pack_dict import Functors.math as fmath from GaudiKernel.SystemOfUnits import GeV @@ -192,13 +191,16 @@ particle_functors = [ def test_functors(alg_name_suffix, functors_to_test, SkipCut=False): - algo = getattr(Configurables, 'InstantiateFunctors__' + alg_name_suffix) - test = algo('Test' + alg_name_suffix) - test.Functions = pack_dict( - {functor.code_repr(): functor - for functor in functors_to_test}) - if not SkipCut: test.Cut = FILTER(ALL) - ApplicationMgr().TopAlg.append(test) + algo = getattr(Algorithms, 'InstantiateFunctors__' + alg_name_suffix) + test = algo( + name='Test' + alg_name_suffix, + Functions={ + functor.code_repr(): functor + for functor in functors_to_test + }, + Cut=FILTER(ALL) if not SkipCut else None) + algs, _ = test.configuration().apply() + ApplicationMgr().TopAlg.append(algs[-1]) def test_pr(prname, functors, only_unwrapped_functors=[]): @@ -213,7 +215,6 @@ app.EvtMax = 0 # these options, so if we *didn't* disable the cache then the test that cling # can handle all of these functors would be bypassed. from Configurables import FunctorFactory -FunctorFactory().UseCache = False # Simple instantiation test: are the templates working? # diff --git a/Phys/FunctorCore/tests/options/test_vector_functors.py b/Phys/FunctorCore/tests/options/test_vector_functors.py index d8fbb6487c9736a2696186a638ee10d71e537c72..c6fec60d620ef1d095330a912b235b9f9552e9a2 100644 --- a/Phys/FunctorCore/tests/options/test_vector_functors.py +++ b/Phys/FunctorCore/tests/options/test_vector_functors.py @@ -10,11 +10,10 @@ ############################################################################### import os import hashlib -import Configurables +from PyConf import Algorithms import GaudiKernel.Configurable from Gaudi.Configuration import (ApplicationMgr, DEBUG) from Functors import (ALL, FILTER, POD) -from Functors.utils import pack_dict from Functors.tests.categories import functors_for_class from Configurables import EvtStoreSvc, FunctorFactory, CondDB, DDDBConf from PyConf.Algorithms import UniqueIDGeneratorAlg, ProducePrFittedForwardTracks @@ -45,16 +44,29 @@ if not inside_cache_generation: def add_algorithm(type_suffix, eval_or_init, functors, producer=None): - algo_type = getattr(Configurables, - 'TestThOrFunctor' + eval_or_init + '__' + type_suffix) + algo_type_str = 'TestThOrFunctor' + eval_or_init + '__' + type_suffix + algo_type = getattr(Algorithms, algo_type_str) + functors = {f.code_repr(): f for f in functors} - algo_instance = algo_type( - Functors=pack_dict(functors), PODFunctors=pack_dict(functors, POD)) - if producer is not None: algo_instance.Input = producer.location - if hasattr(algo_instance, 'Cut'): - algo_instance.Cut = pack_dict({'': FILTER(ALL)}) + kwargs = {} + if producer is not None: + kwargs["Input"] = producer if not inside_cache_generation: - algo_instance.OutputLevel = DEBUG + kwargs["OutputLevel"] = DEBUG + + configurable_algs, _ = algo_type( + Functors=functors, + PODFunctors={k: POD(v) + for k, v in functors.items()}, + **kwargs).configuration().apply() + + # Get the TestThOrFunctor algorithm which we (knowing some PyConf + # externals) should be able to find as the last element in the list + algo_instance = configurable_algs[-1] + # check that the trick from above actually returns the algorithm that we + # are looking for. + if algo_type_str != algo_instance.getType(): + raise Exception("test_vector_functors.py: our trick didn't work") ApplicationMgr().TopAlg.append(algo_instance) diff --git a/Phys/FunctorCore/tests/qmtest/test_functor_datahandle.qmt b/Phys/FunctorCore/tests/qmtest/test_functor_datahandle.qmt new file mode 100644 index 0000000000000000000000000000000000000000..0ee107a4f642ea683c5893f43f841469a4d74e9e --- /dev/null +++ b/Phys/FunctorCore/tests/qmtest/test_functor_datahandle.qmt @@ -0,0 +1,30 @@ + + + + + gaudirun.py + + ../options/functor_datahandle_test.py + + + THOR_DISABLE_JIT=1 + + + +countErrorLines({"FATAL":0, "WARNING":2, "ERROR":0}) + + diff --git a/Phys/FunctorCore/tests/qmtest/test_functor_string_datahandle.qmt b/Phys/FunctorCore/tests/qmtest/test_functor_string_datahandle.qmt new file mode 100644 index 0000000000000000000000000000000000000000..6b709bc4ee3de9ba586bf6c8f88ae3675ceed95d --- /dev/null +++ b/Phys/FunctorCore/tests/qmtest/test_functor_string_datahandle.qmt @@ -0,0 +1,43 @@ + + + + + gaudirun.py + + ../options/functor_datahandle_test.py + + + TEST_FUNCTORS_DH_USE_STRING=1 + + 1 + + +expected_error = 'ERROR TES::Size Usage of DataHandle["/Event/VDP/OutputLocation"] in Functor: TES::Size, requires that owning algorithm FEA contains this TES location inside the ExtraInputs property. This is likely a Configuration/PyConf bug! StatusCode=FAILURE' +if stdout.find(expected_error) == -1: + causes.append("Functor DataHandle check in C++ should have thrown!") + +countErrorLines({"FATAL":1, "WARNING":0, "ERROR":5}) + + diff --git a/Phys/ParticleCombiners/include/CombKernel/ThOrCombiner.h b/Phys/ParticleCombiners/include/CombKernel/ThOrCombiner.h index c639a2409d29e03420c0b3282bbd5f808b8efb85..e96de0b1b0c5b97dec215933e7ae62cc63210209 100644 --- a/Phys/ParticleCombiners/include/CombKernel/ThOrCombiner.h +++ b/Phys/ParticleCombiners/include/CombKernel/ThOrCombiner.h @@ -112,15 +112,12 @@ namespace ThOr::detail::Combiner { Sel::ParticleCombination>>, Sel::ParticleCombination>>; - constexpr static bool is_simd = Backend != SIMDWrapper::Scalar; // Combination12Cut (M = 0), Combination123Cut (M = 1), ..., CombinationCut (M = N-1) template struct Cut { using Signature = mask_b_v( Functors::mask_arg_t, mask_b_v const&, Combination const& ); - constexpr static auto Compilation = - is_simd ? Functors::IFactory::QuietCacheOnly : Functors::IFactory::DefaultCompilationBehaviour; - inline static auto PropertyName = combinationCutName(); - constexpr static auto ExtraHeaders = LHCb::header_map_v>; + inline static auto const PropertyName = combinationCutName(); + constexpr static auto ExtraHeaders = LHCb::header_map_v>; }; }; @@ -130,9 +127,6 @@ namespace ThOr::detail::Combiner { typename LHCb::Event::simd_zip_t::template zip_proxy_type; using Signature = mask_b_v( Functors::mask_arg_t, mask_b_v const&, particle_t const& ); - constexpr static bool is_simd = Backend != SIMDWrapper::Scalar; - constexpr static auto Compilation = - is_simd ? Functors::IFactory::QuietCacheOnly : Functors::IFactory::DefaultCompilationBehaviour; constexpr static auto PropertyName = "CompositeCut"; constexpr static auto ExtraHeaders = LHCb::header_map_v; }; diff --git a/Phys/SelAlgorithms/src/CombineTracksSIMD.h b/Phys/SelAlgorithms/src/CombineTracksSIMD.h index 4b30b8ae555059aa64a78a19fecd23a3b1631e18..b5333849fb040a3857a7fff1f9a963a9a677c8eb 100644 --- a/Phys/SelAlgorithms/src/CombineTracksSIMD.h +++ b/Phys/SelAlgorithms/src/CombineTracksSIMD.h @@ -78,16 +78,13 @@ namespace SelAlgorithms::CombineTracksSIMD { M == 2, Sel::ParticleCombination, boost::mp11::mp_append, M - 1>, Sel::ParticleCombination>>; - constexpr static bool is_simd = Backend != SIMDWrapper::Scalar; // Combination12Cut (M = 0), Combination123Cut (M = 1), ..., CombinationCut (M = N-1) template struct Cut { - using Signature = mask_b_v( Functors::mask_arg_t, mask_b_v const&, + using Signature = mask_b_v( Functors::mask_arg_t, mask_b_v const&, Combination const& ); - constexpr static auto Compilation = - is_simd ? Functors::IFactory::QuietCacheOnly : Functors::IFactory::DefaultCompilationBehaviour; - inline static auto PropertyName = combinationCutName(); - constexpr static auto ExtraHeaders = LHCb::header_map_v>; + inline static auto const PropertyName = combinationCutName(); + constexpr static auto ExtraHeaders = LHCb::header_map_v>; }; }; @@ -98,9 +95,6 @@ namespace SelAlgorithms::CombineTracksSIMD { LHCb::Pr::ProxyBehaviour::Contiguous, LHCb::Event::Composites const>; using Signature = mask_b_v( Functors::mask_arg_t, mask_b_v const&, particle_t const& ); - constexpr static bool is_simd = Backend != SIMDWrapper::Scalar; - constexpr static auto Compilation = - is_simd ? Functors::IFactory::QuietCacheOnly : Functors::IFactory::DefaultCompilationBehaviour; constexpr static auto PropertyName = "VertexCut"; constexpr static auto ExtraHeaders = LHCb::header_map_v; }; diff --git a/Phys/SelAlgorithms/src/TestFunctors.h b/Phys/SelAlgorithms/src/TestFunctors.h index b84856ffb2035d54d081045fa68c568c4918d19f..e5e5304c2c414cb772f95e19a90c381781ec8aee 100644 --- a/Phys/SelAlgorithms/src/TestFunctors.h +++ b/Phys/SelAlgorithms/src/TestFunctors.h @@ -80,7 +80,7 @@ namespace ThOr::Functors::Tests::detail { /** Take some functors that act on elements of the input. */ - template + template struct Dump { static_assert( Target == SIMDWrapper::Scalar || !WrappedInPOD ); template @@ -90,11 +90,8 @@ namespace ThOr::Functors::Tests::detail { constexpr static bool IsAFilterTag = false; constexpr static auto VectorBackend = Target; constexpr static bool ShouldGivePOD = WrappedInPOD; - constexpr static bool ShouldBeJITed = !InFunctorCache; constexpr static auto PropertyName = WrappedInPOD ? "PODFunctors" : "Functors"; - constexpr static auto Compilation = - InFunctorCache ? ::Functors::IFactory::QuietCacheOnly : ::Functors::IFactory::QuietJITOnly; - constexpr static auto ExtraHeaders = []() { + constexpr static auto ExtraHeaders = []() { if constexpr ( sizeof...( ConsumedTypes ) > 0 ) { return ( LHCb::header_map_v> + ... ) + LHCb::header_map_v; } else { @@ -102,8 +99,7 @@ namespace ThOr::Functors::Tests::detail { } }(); static std::string name() { - return SIMDWrapper::instructionSetName( Target ) + " " + ( InFunctorCache ? "cache" : "cling" ) + - ( WrappedInPOD ? " POD" : "" ); + return SIMDWrapper::instructionSetName( Target ) + " " + ( WrappedInPOD ? " POD" : "" ); } // Possible expected return types template @@ -112,26 +108,18 @@ namespace ThOr::Functors::Tests::detail { /** For completeness, also take a functor that filters an entire container. */ - template + template struct Filter { using iterable_t = typename LHCb::Event::simd_zip_t; using Signature = ::Functors::filtered_t( iterable_t const& ); constexpr static bool IsAFilterTag = true; constexpr static auto VectorBackend = Target; - constexpr static bool ShouldBeJITed = !InFunctorCache; constexpr static auto PropertyName = "Cut"; constexpr static auto ExtraHeaders = LHCb::header_map_v; - constexpr static auto Compilation = - InFunctorCache ? ::Functors::IFactory::QuietCacheOnly : ::Functors::IFactory::QuietJITOnly; - static std::string name() { - return SIMDWrapper::instructionSetName( Target ) + " " + ( InFunctorCache ? "cache" : "cling" ) + " filter"; - } + static std::string name() { return SIMDWrapper::instructionSetName( Target ) + " " + " filter"; } }; /** Construct the base type -- defining an alias here saves typing below. - * Note that having Dump (true => load from functor - * cache) last is important, as it encourages functors to be JITed before a - * cache is loaded. */ template struct base_helper { @@ -141,23 +129,15 @@ namespace ThOr::Functors::Tests::detail { // mp_list> if ConsumedTypeList = {T}, // otherwise mp_list> using base_class = boost::mp11::mp_apply; - template - using first_filter_helper = Filter; - // mp_list> if InstantiatedTypeList = {T}, - // otherwise mp_list<> - using first_filter = boost::mp11::mp_transform; template - using first_dumps_helper = - boost::mp11::mp_list, // scalar, no POD, JIT - Dump, // scalar, POD, JIT - Dump>; // scalar, POD, cache + using first_dumps_helper = boost::mp11::mp_list>; // mp_list, Dump<...>, Dump<...>> where each Dump<...> received // the contents InstantiatedTypeList as a trailing parameter pack using first_dumps = boost::mp11::mp_apply; template struct second_dumps_helper { template - using fn = Dump; + using fn = Dump; }; // mp_list, // Dump, @@ -173,15 +153,14 @@ namespace ThOr::Functors::Tests::detail { using second_dumps = boost::mp11::mp_transform_q, target_vector_backends>; template - using second_filters_helper = Filter; + using filters_helper = Filter; // Empty list if InstantiatedTypeList is empty (void functor), otherwise // mp_list, // Filter, // ...> // for distinct_vector_backends = {Target0, Target1, ...} - using second_filters = - boost::mp11::mp_product; - using all_args = boost::mp11::mp_append; + using filters = boost::mp11::mp_product; + using all_args = boost::mp11::mp_append; using type = boost::mp11::mp_apply; }; template @@ -331,15 +310,6 @@ namespace ThOr::Functors::Tests { continue; } if ( ( !std::get( functors ) || ... ) ) { throw exception( "A subset of functors were null" ); } - // Check that the JITed functors were JITed and the functors from the - // cache were not - std::array const was_jitted{std::get( functors ).wasJITCompiled()...}, - should_be_jitted{dump_tags::ShouldBeJITed...}; - if ( was_jitted != should_be_jitted ) { - std::ostringstream oss; - oss << "Got was_jitted = " << was_jitted << ", should_be_jitted = " << should_be_jitted; - throw exception( oss.str() ); - } // Get all of the return types as strings std::array const functor_rtypes{System::typeinfoName( std::get( functors ).rtype() )...}; // We also allow all functor return types that are trivially copyable (e.g. enums) diff --git a/Phys/SelAlgorithms/tests/options/test_dump_container.py b/Phys/SelAlgorithms/tests/options/test_dump_container.py index e9f7761976d4ff4d3ee8ea5486d0d206a7efbb73..aa5ae58e28661495c483525e9f8bba3438d45473 100644 --- a/Phys/SelAlgorithms/tests/options/test_dump_container.py +++ b/Phys/SelAlgorithms/tests/options/test_dump_container.py @@ -10,7 +10,6 @@ ############################################################################### from Gaudi.Configuration import ApplicationMgr from Functors import (PT, ETA, PHI, POD) -from Functors.utils import pack_dict from Configurables import (EvtStoreSvc, ProducePrFittedForwardTracks, DumpContainer__PrFittedForwardTracks, UniqueIDGeneratorAlg) @@ -20,11 +19,11 @@ unique_id_gen = UniqueIDGeneratorAlg() produce = ProducePrFittedForwardTracks(Output='Fake/Tracks') consume = DumpContainer__PrFittedForwardTracks( Input=produce.Output, - Branches=pack_dict({ - 'PT': PT, - 'ETA': ETA, - 'PHI': PHI - }, wrap=POD), + Branches={ + 'PT': POD(PT), + 'ETA': POD(ETA), + 'PHI': POD(PHI) + }, VoidBranches={}, DumpFileName='DumpContainer.root', DumpTreeName='DumpTree') diff --git a/Phys/SelTools/include/SelTools/SigmaNet.h b/Phys/SelTools/include/SelTools/SigmaNet.h index b79d87935ebf2f23bb37ba46db3cb33d516d6fd9..260d35f35e6d5a6c00c33b442e65da783fb94543 100644 --- a/Phys/SelTools/include/SelTools/SigmaNet.h +++ b/Phys/SelTools/include/SelTools/SigmaNet.h @@ -12,9 +12,12 @@ #include "GaudiKernel/Environment.h" #include "GaudiKernel/GaudiException.h" #include "Kernel/STLExtensions.h" -#include "SelTools/MVAUtils.h" +#include "MVAUtils.h" +#include +#include #include +#include #include #include @@ -30,14 +33,14 @@ namespace Sel { return std::any_cast( i->second ); } - void groupsort2( LHCb::span x ) { + inline void groupsort2( LHCb::span x ) { for ( long unsigned int i = 0; i + 1 < x.size(); i += 2 ) { if ( x[i] > x[i + 1] ) std::swap( x[i], x[i + 1] ); } } - void multiply_and_add( LHCb::span w, LHCb::span b, LHCb::span x, - LHCb::span y ) { + inline void multiply_and_add( LHCb::span w, LHCb::span b, LHCb::span x, + LHCb::span y ) { assert( w.size() == y.size() * x.size() ); assert( b.size() == y.size() ); for ( long unsigned int i = 0; i < y.size(); ++i ) { @@ -45,7 +48,7 @@ namespace Sel { } } - float sigmoid( float x ) { return 1 / ( 1 + expf( -1 * x ) ); } + inline float sigmoid( float x ) { return 1 / ( 1 + expf( -1 * x ) ); } template float evaluate( LHCb::span values, LHCb::span weights, LHCb::span biases, @@ -54,7 +57,7 @@ namespace Sel { assert( values.size() == layer_sizes.front() ); - std::array storage; + std::array storage{}; auto input_for = [&]( long unsigned int layer ) { assert( layer > 0 && layer <= layer_sizes.size() ); assert( layer_sizes[layer - 1] <= N ); @@ -134,13 +137,13 @@ namespace Sel { void setupReader(); }; - auto SigmaNet::operator()( LHCb::span values ) const { + inline auto SigmaNet::operator()( LHCb::span values ) const { if ( values.size() != m_input_size ) { throw exception( "'InputSize' does not agree with provided Input!!!" ); } // Evaluate MVA return evaluate( values, m_weights, m_biases, m_layer_sizes, m_monotone_constraints, m_lambda ); } - void SigmaNet::bind( Gaudi::Algorithm* alg ) { + inline void SigmaNet::bind( Gaudi::Algorithm* alg ) { readConfig( *alg ); // Retrieve the configuration readWeightsFile( *alg ); // Read the .pkl File with the weights if ( alg->msgLevel( MSG::VERBOSE ) ) { @@ -151,7 +154,7 @@ namespace Sel { } // Retrieve the configuration - void SigmaNet::readConfig( Gaudi::Algorithm const& alg ) { + inline void SigmaNet::readConfig( Gaudi::Algorithm const& alg ) { if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "Start reading config..." << endmsg; } @@ -247,10 +250,9 @@ namespace Sel { if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "weightfile: " << unresolved_weightfile << endmsg; } System::resolveEnv( unresolved_weightfile, m_weightfile ).ignore(); // LEON - return; } - void SigmaNet::readWeightsFile( Gaudi::Algorithm const& alg ) { + inline void SigmaNet::readWeightsFile( Gaudi::Algorithm const& alg ) { // Check that the weightfile exists if ( !checkWeightsFile() ) { throw exception( "Couldn't open file: " + m_weightfile ); } @@ -267,10 +269,10 @@ namespace Sel { m_layer_sizes = std::vector( m_n_layers + 1, 0 ); - nlohmann::json jf = nlohmann::json::parse( fin ); + auto jf = nlohmann::json::parse( fin ); for ( auto it = jf.begin(); it != jf.end(); ++it ) { - std::string current_key = it.key(); + const std::string& current_key = it.key(); if ( current_key.find( "sigmanet.sigma" ) != std::string::npos ) { if ( jf.at( current_key ).at( 0 ) != m_lambda ) { @@ -338,7 +340,7 @@ namespace Sel { } } - bool SigmaNet::checkWeightsFile() { + inline bool SigmaNet::checkWeightsFile() { // Check existence of WeightFile: locally return std::ifstream{m_weightfile}.good(); }