diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 975126d12339ea8f9ed247f713c5f74d7dcf88f4..1501966e3d9adedf6992bef295df56f9f519d390 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ build: - export CMAKE_PREFIX_PATH=${build_path}:${CMAKE_PREFIX_PATH} - export LCG_VERSION=$(jq -r '.projects | map(select(.name == "LCG")) | .[0].version' $build_path/slot-config.json) - lb-project-init - - make CMAKEFLAGS=-DLOKI_BUILD_FUNCTOR_CACHE=OFF + - make CMAKEFLAGS=-DBUILD_FUNCTOR_CACHE=OFF - ln -s ./build.$BINARY_TAG/run run # Allow failure as the nightly slot we depend on may have itself failed allow_failure: true diff --git a/MooreCache/CMakeLists.txt b/MooreCache/CMakeLists.txt index 813172a8a7608cfc6bfd41aed153ac6435d127b1..7ef3b4506287b2ff4045b05ddbe1c8ebf69cf401 100644 --- a/MooreCache/CMakeLists.txt +++ b/MooreCache/CMakeLists.txt @@ -13,121 +13,29 @@ MooreCache ---------- #]=======================================================================] -# Import the cache creation module -include(LoKiFunctorsCache) +# Import the cache creation module (function build_functor_cache) +include(FunctorsCache) -set(cache_deps - LHCb::LoKiCoreLib - LHCb::PhysEvent - Rec::FunctorCoreLib # this should bring most dependencies indirectly - Rec::ParticleCombinersLib - # TODO can we get this list automatically? - # TODO can we depend only on the headers and not the .so files? +build_functors_cache(Moore_FunctorCache_Hlt1_hlt1_pp_default + ${PROJECT_SOURCE_DIR}/Hlt/Moore/tests/options/mdf_input_and_conds.py + ${PROJECT_SOURCE_DIR}/Hlt/Hlt1Conf/options/hlt1_pp_default.py ) -if(TARGET ${PROJECT_NAME}_MergeConfDB2) - list(APPEND cache_deps ${PROJECT_NAME}_MergeConfDB2) -elseif(TARGET MergeConfDB2) - # backward compatibility - list(APPEND cache_deps MergeConfDB2) -endif() -# inter-project dependencies for super project build -# FIXME Allen is not strictly needed, however, it seems that -# it's being written and read at the same time and that causes problems. -foreach(project Gaudi LHCb Lbcom Rec Allen) - foreach(merge_target MergeComponents MergeRootmaps MergeConfdb MergeConfDB2) - if (TARGET ${project}_${merge_target}) - list(APPEND cache_deps ${project}_${merge_target}) - endif() - endforeach() -endforeach() - -# Disable LoKi-specific hacks in LoKiFunctorsCachePostActionOpts.py -set(LOKI_FUNCTORS_CACHE_POST_ACTION_OPTS) - -list(APPEND hlt1_settings hlt1_pp_default) list(APPEND hlt2_settings - options/hlt2_pp_2025 - options/hlt2_pp_thor - options/sprucing/spruce_production - ) - -foreach(name IN LISTS hlt1_settings) - # note that we don't use DisableLoKiCacheFunctors.py from Rec since it is not installed (with - # the python package) and is thus hard to get reliably. - loki_functors_cache(Moore_FunctorCache_Hlt1_${name} - ${CMAKE_CURRENT_SOURCE_DIR}/options/DisableLoKiCacheFunctors.py - ${PROJECT_SOURCE_DIR}/Hlt/Moore/tests/options/mdf_input_and_conds.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/process_zero_events.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/silence_application_manager.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/ThOr_create_cache_opts.py - ${PROJECT_SOURCE_DIR}/Hlt/Hlt1Conf/options/${name}.py - FACTORIES FunctorFactory - LINK_LIBRARIES - Gaudi::GaudiKernel - Rec::FunctorCoreLib - DEPENDS ${cache_deps} - # no reason to split much becasue hlt1_pp_default currently only has O(20) functors - SPLIT 1 - ) - if(LOKI_BUILD_FUNCTOR_CACHE) - set_property(TARGET Moore_FunctorCache_Hlt1_${name} PROPERTY DEFINE_SYMBOL "Moore_FunctorCache_EXPORTS") - endif() -endforeach() - -# Limit number of functor cache compilations so that we use at most 3/4 of physical RAM. -# Note that the number of jobs is determined at configuration time so if you configure -# and build on different machines, the setting might be suboptimal. -cmake_host_system_information(RESULT total_ram QUERY TOTAL_PHYSICAL_MEMORY) -set(average_ram_usage 4000) # approximate RAM usage of one compilation job in MB -math(EXPR max_cache_compile_jobs "${total_ram}*1/2/${average_ram_usage}" OUTPUT_FORMAT DECIMAL) -if(NOT "${max_cache_compile_jobs}" GREATER_EQUAL "1") - set(max_cache_compile_jobs 1) -endif() -set(MAX_CACHE_COMPILE_JOBS ${max_cache_compile_jobs} CACHE STRING "Size of the functor cache compilation pool") -set_property(GLOBAL APPEND PROPERTY JOB_POOLS functor_cache_compile_pool=${MAX_CACHE_COMPILE_JOBS}) + options/hlt2_pp_2025 + options/hlt2_pp_thor + options/sprucing/spruce_production +) foreach(options_path IN LISTS hlt2_settings) # Replace forward slashes with underscores to form a valid CMake identifier string(REPLACE "/" "_" name ${options_path}) set(cache_name Moore_FunctorCache_Hlt2_${name}) - loki_functors_cache(${cache_name} - ${CMAKE_CURRENT_SOURCE_DIR}/options/DisableLoKiCacheFunctors.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/ThOr_create_cache_opts.py + build_functors_cache(${cache_name} ${PROJECT_SOURCE_DIR}/Hlt/Moore/tests/options/mdf_input_and_conds_hlt2.py ${CMAKE_CURRENT_SOURCE_DIR}/options/output_mdf_streams.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/process_zero_events.py - ${CMAKE_CURRENT_SOURCE_DIR}/options/silence_application_manager.py ${PROJECT_SOURCE_DIR}/Hlt/Hlt2Conf/${options_path}.py - FACTORIES - FunctorFactory - LINK_LIBRARIES - Gaudi::GaudiKernel - Rec::FunctorCoreLib - Rec::ParticleCombinersLib - DEPENDS ${cache_deps} SPLIT 25 ) - if(LOKI_BUILD_FUNCTOR_CACHE) - set_property(TARGET ${cache_name} PROPERTY DEFINE_SYMBOL "Moore_FunctorCache_EXPORTS") - set_property(TARGET ${cache_name} PROPERTY JOB_POOL_COMPILE functor_cache_compile_pool) - endif() endforeach() - -# use precompiled headers for functor caches -# use first HLT2 lib, aka pch_lib to precompile and then reuse the precompiled headers for others -if(LOKI_BUILD_FUNCTOR_CACHE) - find_file(JIT_include_file NAMES "Functors/JIT_includes.h") - list(POP_FRONT hlt2_settings hlt2_head) - string(REPLACE "/" "_" hlt2_cache_name ${hlt2_head}) - set(pch_lib Moore_FunctorCache_Hlt2_${hlt2_cache_name}) - target_precompile_headers(${pch_lib} PRIVATE ${JIT_include_file}) - foreach(name IN LISTS hlt1_settings) - target_precompile_headers(Moore_FunctorCache_Hlt1_${name} REUSE_FROM ${pch_lib}) - endforeach() - foreach(options_path IN LISTS hlt2_settings) - string(REPLACE "/" "_" name ${options_path}) - target_precompile_headers(Moore_FunctorCache_Hlt2_${name} REUSE_FROM ${pch_lib}) - endforeach() -endif() diff --git a/MooreCache/options/DisableLoKiCacheFunctors.py b/MooreCache/options/DisableLoKiCacheFunctors.py deleted file mode 100644 index 08c4a3b9c85e6c11495ad76c7965889b2c6e2533..0000000000000000000000000000000000000000 --- a/MooreCache/options/DisableLoKiCacheFunctors.py +++ /dev/null @@ -1,14 +0,0 @@ -############################################################################### -# (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -from Configurables import ApplicationMgr - -ApplicationMgr().Environment["LOKI_DISABLE_CACHE"] = "1" -ApplicationMgr().Environment["THOR_DISABLE_JIT"] = "1" diff --git a/MooreCache/options/ThOr_create_cache_opts.py b/MooreCache/options/ThOr_create_cache_opts.py deleted file mode 100644 index c3047aa4371c2f76208aaefc2ab9710702fcd136..0000000000000000000000000000000000000000 --- a/MooreCache/options/ThOr_create_cache_opts.py +++ /dev/null @@ -1,16 +0,0 @@ -############################################################################### -# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -from Configurables import ApplicationMgr - -ApplicationMgr().Environment["THOR_DISABLE_JIT"] = "1" -ApplicationMgr().Environment["THOR_DISABLE_CACHE"] = "1" -ApplicationMgr().Environment["THOR_JIT_LIBDIR"] = "." -ApplicationMgr().Environment["THOR_JIT_EXTRA_ARGS"] = "" diff --git a/MooreCache/options/silence_application_manager.py b/MooreCache/options/silence_application_manager.py deleted file mode 100644 index 450f7981c9905a10e5e6c657c84b584e499ab9e1..0000000000000000000000000000000000000000 --- a/MooreCache/options/silence_application_manager.py +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################### -# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -import logging - -from Gaudi.Configuration import ERROR -from Moore import options - -options.output_level = ERROR -options.python_logging_level = logging.ERROR