diff --git a/Hlt/Hlt2Conf/options/hlt2_test_flagging.py b/Hlt/Hlt2Conf/options/hlt2_test_flagging.py index fb0a89183cd167a2dfcfdede7a7a8d1b57245555..ab6fbff0f5cb4fed3a2e871412c346d8620b4995 100644 --- a/Hlt/Hlt2Conf/options/hlt2_test_flagging.py +++ b/Hlt/Hlt2Conf/options/hlt2_test_flagging.py @@ -27,6 +27,7 @@ options.output_file = "hlt2_test_production.dst" options.output_manifest_file = "hlt2_test_production.tck.json" options.output_type = "ROOT" options.root_ioalg_name = "RootIOAlgExt" +options.dstformat = "DST" def make_lines(make_lines=None): diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/check_output.py b/Hlt/Hlt2Conf/python/Hlt2Conf/check_output.py index 2088dc942e8d0f8f27f879108396f4c7054c1b04..0fff06e81414cee8938172a7f7326352b63b47dc 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/check_output.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/check_output.py @@ -55,7 +55,7 @@ def check_persistreco(TES, locations, N=3, unexpected_locs=[]): ) -def check_MCoutput(TES, RECO_ROOT, fs=2, mcpart=100): +def check_MCoutput(TES, RECO_ROOT, fs=2, mcpart=100, dstformat="DST"): """ Check MC TES locations are present for dst workflows @@ -72,28 +72,33 @@ def check_MCoutput(TES, RECO_ROOT, fs=2, mcpart=100): f"MC relations table for charged particles size is {MC_charged_rel.relations().size()}" ) - MC_part = TES[RECO_ROOT + "/MC/Particles"] + if dstformat != "uDST": + MC_ROOT = "/Event" + else: + MC_ROOT = RECO_ROOT + MC_part = TES[MC_ROOT + "/MC/Particles"] + if not MC_part: error("MC particles not propagated") if not MC_part.size() > mcpart: error("MC particles not correctly propagated") - print(f"MC particles {RECO_ROOT}/MC/Particles container has size {MC_part.size()}") + print(f"MC particles {MC_ROOT}/MC/Particles container has size {MC_part.size()}") - MC_part_packed = TES[f"{RECO_ROOT}/pSim/MCParticles"].mcParts().size() + MC_part_packed = TES[f"{MC_ROOT}/pSim/MCParticles"].mcParts().size() print(f"Packed MC particles container has size {MC_part_packed}") if not MC_part.size() == MC_part_packed: error( f"MC object packing not working: {MC_part.size()} unpacked, vs. {MC_part_packed} packed" ) - MC_vert = TES[RECO_ROOT + "/MC/Vertices"] + MC_vert = TES[MC_ROOT + "/MC/Vertices"] if not MC_vert: error("MC vertices not propagated") if not MC_vert.size() > mcpart: error("MC vertices not correctly propagated") - print(f"MC vertices {RECO_ROOT}/MC/Vertices container has size {MC_vert.size()}") + print(f"MC vertices {MC_ROOT}/MC/Vertices container has size {MC_vert.size()}") - MC_vert_packed = TES[f"{RECO_ROOT}/pSim/MCVertices"].mcVerts().size() + MC_vert_packed = TES[f"{MC_ROOT}/pSim/MCVertices"].mcVerts().size() print(f"Packed MC vertices container has size {MC_vert_packed}") if MC_vert.size() != MC_vert_packed: error( diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py index 038b25c82d7a955be66ed7b1d443f2b15a7c795e..cc1291e0735dcbaf57c4346bd314af9d6e4d71e1 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py @@ -75,6 +75,7 @@ options.root_ioalg_name = "RootIOAlgExt" options.evt_max = args.n if args.n else -1 options.gaudipython_mode = True options.input_stream = args.s +options.dstformat = "DST" config = configure_input(options) input_process = args.p @@ -142,7 +143,7 @@ for ii in range(nevents): # Check MC locations if simulation like if "_dstinput" in args.i: - check_MCoutput(TES, RECO_ROOT) + check_MCoutput(TES, RECO_ROOT, dstformat="DST") # Check rawbanks specific to this stream print("args.rb ", args.rb) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 908f386870a2bf07e1c77ab2cb0c7e25fee1635c..74d827a884b4e4005dc37a22073e739ecdcf08d6 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -17,7 +17,7 @@ from GaudiConf.LbExec import TestOptionsBase from PyConf.application import ROOT_KEY from PyConf.packing import persistreco_version from PyConf.reading import reconstruction as reconstruction_reading -from PyConf.reading import tes_root +from PyConf.reading import tes_root, tes_root_mc from PyConf.reading import upfront_reconstruction as upfront_reconstruction_reading from RecoConf.reconstruction_objects import reconstruction @@ -52,9 +52,11 @@ class Options(DefaultOptions): - `options.process` is set to "Spruce" to control the reconstruction control flow (ie. get it from input file). This is not required if set to "TurboPass" - `process` is set to "spruce" or "pass" based on the lines to run for overall Moore control flow in config.py https://gitlab.cern.ch/lhcb/Moore/-/blob/master/Hlt/Moore/python/Moore/config.py#L298 """ + tes_root_mc.global_bind(dstformat=self.dstformat) if self.input_process: reconstruction_reading.global_bind(input_process=self.input_process) tes_root.global_bind(input_process=self.input_process) + tes_root_mc.global_bind(input_process=self.input_process) reconstruction_reading.global_bind( simulation=self.simulation and self.input_type == ROOT_KEY diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 3a2346c732179dff89f0b2cbe469954d2d03bc42..95bd9fe3983c3e7ddaefe5cd8e51247bbeb21ebe 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -14,6 +14,7 @@ import sys from functools import partial from Allen.config import run_allen_reconstruction # noqa: F401 +from GaudiConf.LbExec import DSTFormatTypes from PyConf import configurable from PyConf.Algorithms import ( BandwidthMonitor, @@ -162,7 +163,12 @@ def moore_control_flow(options, streams, process, analytics=False): output_manifest_file=options.output_manifest_file, output_prefix=event_output_prefix, reco_output_prefix=reco_output_prefix, - clone_mc=options.simulation and options.input_type == "ROOT", + # in the updated work flow, the /Event/HLT2/MC/Particles are only created for the microdst + # TODO: Instead of saving the MCParticles to /Event/MC/Particles for the normal DST, create some hyperlinks in /Event/HLT2/MC/Particles + clone_mc=options.simulation + and options.input_type == "ROOT" + and options.dstformat == DSTFormatTypes.uDST, + clone_mc_relation=options.simulation and options.input_type == "ROOT", ) ) diff --git a/Hlt/Moore/python/Moore/persistence/__init__.py b/Hlt/Moore/python/Moore/persistence/__init__.py index 6d55998e03b65c2225c79e5abdf22a6cf29ad234..e21f624c2336d2fb459678926d4849019129a1f3 100644 --- a/Hlt/Moore/python/Moore/persistence/__init__.py +++ b/Hlt/Moore/python/Moore/persistence/__init__.py @@ -112,6 +112,7 @@ def persist_line_outputs( output_prefix=DEFAULT_OUTPUT_PREFIX, # this is where everything goes reco_output_prefix=DEFAULT_OUTPUT_PREFIX, # this is where reco objects come from clone_mc=False, + clone_mc_relation=False, enable_packing_checks=False, enable_checksum=False, allow_missing_containers=False, @@ -139,7 +140,7 @@ def persist_line_outputs( dhs_for_hlt2_tistos = [] algs_for_hlt2_tistos = [] locations_for_hlt2_tistos = [] - if "Spruce" in output_prefix and clone_mc: # Sprucing case + if "Spruce" in output_prefix and clone_mc_relation: # Sprucing case assert associate_mc is False, ( "Sprucing does not support MC association. This is done at the HLT2 step." ) @@ -208,6 +209,19 @@ def persist_line_outputs( locations.update(mc_locations_mapping.values()) mc_packer_cf.children = mc_cloner + mc_packer_cf.children cf.append(mc_packer_cf) + elif associate_mc or clone_mc_relation: + # In HLT2 reconstruction, the MCParticle relation table is always built with + # the TES path `/Event/MC/Particles`. So here we specify the `MCParticlesLocation` + # by hand instead of using `tes_root_mc()` function + MCParticlesLocation = "/Event/MC/Particles" + mc_locations_mapping = dict( + (l, prefix(l, "/Event")) + for l in ( + MCParticlesLocation, + MCParticlesLocation.replace("Particles", "Vertices"), + ) + ) + locations.update(mc_locations_mapping.values()) ##TODO: replace "locations" with "requested" determined below... encoding_key = int( diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index e344245cb1c7b43624a8f5123f66979a49132044..45545c87a052617606dae1260003be2071f25cec 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -57,7 +57,6 @@ def hlt2(options: Options, *raw_args): settings(default=None, will run all lines in one stream): to set the streams and reconstruction options flagging(default=False): to run MC in flagging mode require_deployed_trigger_key(default=False) - flagging(default=False): to run MC in flagging mode persistreco(default=False): to run MC in persistreco mode (with flaggging only) rawbanks(default=None): to run MC persisting rawbanks (with flaggging only) reco_only(default=False): to run MC in reconstruction only mode (with flaggging only) @@ -173,6 +172,7 @@ def _hlt2( without_ut=False, trkeff_probe_matching="2025-like", ): + from GaudiConf.LbExec import DSTFormatTypes from PyConf.Algorithms import MuonProbeToLongMatcher from PyConf.application import metainfo_repos, retrieve_encoding_dictionary from RecoConf import mc_checking @@ -181,6 +181,12 @@ def _hlt2( from Moore import run_moore + if ( + (options.dstformat if hasattr(options, "dstformat") else DSTFormatTypes.DST) + == DSTFormatTypes.uDST + ) and flagging: + raise ConfigurationError("microdst and flagging options are conflict") + if require_deployed_trigger_key: metainfo_repos.global_bind(repos=[]) # only use repos on cvmfs retrieve_encoding_dictionary.global_bind( diff --git a/Hlt/Moore/python/Moore/stream_writers.py b/Hlt/Moore/python/Moore/stream_writers.py index b6520d96d4fb351d50939789a8b76f5f8c55a016..5d725a231d1b42815bbd3d7d4ce8a45b7a80af7b 100644 --- a/Hlt/Moore/python/Moore/stream_writers.py +++ b/Hlt/Moore/python/Moore/stream_writers.py @@ -9,8 +9,10 @@ # or submit itself to any jurisdiction. # ############################################################################### import logging +import os from shutil import copy +from GaudiConf.LbExec import DSTFormatTypes from PyConf import configurable from PyConf.Algorithms import ( AddressKillerAlg, @@ -30,6 +32,12 @@ from PyConf.application import ( root_writer, ) from PyConf.components import force_location +from PyConf.persistency_locations import ( + default_persistreco_version as _default_persistreco_version, +) +from PyConf.persistency_locations import ( + digi_to_mc_particles_locations, +) from PyConf.utilities import ConfigurationError from .config_tools import unique @@ -60,6 +68,25 @@ sim_veto_list = [ "/Event/DAQ/RawEvent", ] +microdst_locations = [ + "/Event/HLT2/pSim/MCParticles", + "/Event/HLT2/pSim/MCVertices", + "/Event/MC/Header", + "/Event/MC/OriginalHeader", + "/Event/MC/TrackInfo", +] + +dst_locations = [ + "/Event/MC/Header", + "/Event/MC/TrackInfo", + "/Event/pSim/MCParticles", + "/Event/pSim/MCVertices", + "/Event/Gen/Header", + "/Event/Gen/BeamParameters", + "/Event/MC/DigiHeader", + "/Event/pSim/Rich/DigitSummaries", +] + @configurable def stream_writer( @@ -174,7 +201,28 @@ def stream_writer( # This part is only meant for simulation # always false for data and by default true for simulation - if write_all_input_leaves and propagate_mc: + + if options.dstformat == DSTFormatTypes.uDST: + # Make the entire datastore anonymous. All persistent info of all addresses is entirely removed. + # otherwise, root_writer saves a link to the input event/file + # Since the MC particles from HLT2 are not packed into RawEvent, should be declared here + writers.append(AddressKillerAlg()) + if process == "spruce": + microdst_locations.append("/Event/Spruce/HLT2/pSim/MCParticles") + microdst_locations.append("/Event/Spruce/HLT2/pSim/MCVertices") + writers.append( + root_writer(full_fname, [output_location] + microdst_locations) + ) + + elif options.dstformat == DSTFormatTypes.DST: + writers.append(AddressKillerAlg()) + # only one version for the locations at current, so use the default one + version = _default_persistreco_version + for key, value in digi_to_mc_particles_locations[version].items(): + if key not in ["HcalDigitsV1", "EcalDigitsV1"]: + dst_locations.append("/Event/Link/Raw/" + value) + writers.append(root_writer(full_fname, [output_location] + dst_locations)) + elif write_all_input_leaves and propagate_mc: # Make the entire datastore anonymous. All persistent info of all addresses is entirely removed. # otherwise, root_writer saves a link to the input event/file writers.append(AddressKillerAlg()) diff --git a/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_dstformat.py b/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_dstformat.py new file mode 100644 index 0000000000000000000000000000000000000000..b0df7c932b31050ee9cfa128761888ac1b116a0e --- /dev/null +++ b/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_dstformat.py @@ -0,0 +1,42 @@ +############################################################################### +# (c) Copyright 2024 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +"""Options for running HLT2 lines using microdst option""" + +from Hlt2Conf.lines.test.hlt2_test import hlt2_test_lines +from Hlt2Conf.lines.topological_b import all_lines +from Moore import Options, run_moore +from Moore.production import hlt2 +from RecoConf.decoders import default_VeloCluster_source +from RecoConf.global_tools import ( + stateProvider_with_simplified_geom, + trackMasterExtrapolator_with_simplified_geom, +) +from RecoConf.hlt2_global_reco import make_light_reco_pr_kf +from RecoConf.hlt2_global_reco import reconstruction as hlt2_reconstruction +from RecoConf.reconstruction_objects import reconstruction + + +def run_hlt2(options: Options): + def make_lines(make_lines=None): + if make_lines: + return make_lines() + builders = list(hlt2_test_lines.values()) + list(all_lines.values()) + return [builder() for builder in builders] + + public_tools = [ + trackMasterExtrapolator_with_simplified_geom(), + stateProvider_with_simplified_geom(), + ] + with ( + reconstruction.bind(from_file=False), + hlt2_reconstruction.bind(make_reconstruction=make_light_reco_pr_kf), + ): + return run_moore(options, make_lines, public_tools) diff --git a/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_spruce_dstformat.py b/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_spruce_dstformat.py new file mode 100644 index 0000000000000000000000000000000000000000..da5c453b277b053fd97b80cff30ce771db465aa8 --- /dev/null +++ b/Hlt/Moore/python/Moore/tests/lhcbintegrationtests_options_spruce_dstformat.py @@ -0,0 +1,40 @@ +############################################################################### +# (c) Copyright 2025 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +"""Test running Sprucing line on output of topo{2,3} persistreco hlt2 lines. + +Input from HLT2 is DST as will be the case for simulation. + +Produces spruce_dstinput.dst +""" + +from Hlt2Conf.lines.b_to_charmonia import sprucing_lines +from Hlt2Conf.lines.test.spruce_test import Test_sprucing_line +from Moore import Options, run_moore +from PyConf.reading import reconstruction as reco_spruce +from PyConf.reading import upfront_reconstruction as upfront_spruce +from RecoConf.global_tools import stateProvider_with_simplified_geom +from RecoConf.reconstruction_objects import reconstruction + + +def run_spruce(options: Options): + def make_lines(): + builders = [builder() for builder in sprucing_lines.values()] + builders.append(Test_sprucing_line(name="Spruce_Test_line")) + return builders + + public_tools = [stateProvider_with_simplified_geom()] + + with ( + reconstruction.bind(from_file=True, spruce=True), + reco_spruce.bind(simulation=True), + upfront_spruce.bind(simulation=True), + ): + return run_moore(options, make_lines, public_tools) diff --git a/Hlt/Moore/tests/options/starterkit/first-analysis-steps/interactive-dst.py b/Hlt/Moore/tests/options/starterkit/first-analysis-steps/interactive-dst.py index 50f2129991ebec892eeaa7636a46b54ca1d1995f..abefca1c60943b0e947e666955a77231312a5e8d 100644 --- a/Hlt/Moore/tests/options/starterkit/first-analysis-steps/interactive-dst.py +++ b/Hlt/Moore/tests/options/starterkit/first-analysis-steps/interactive-dst.py @@ -15,6 +15,7 @@ import sys import GaudiPython as GP from Moore import options from PyConf.application import configure, configure_input +from PyConf.reading import tes_root, tes_root_mc LHCb = GP.gbl.LHCb import argparse @@ -94,9 +95,13 @@ options.gaudipython_mode = True options.input_stream = args.input_stream options.geometry_version = "run3/trunk" options.conditions_version = "master" +options.dstformat = "uDST" config = configure_input(options) +tes_root_mc.global_bind(dstformat="uDST", input_process=args.input_process) +# tes_root.global_bind(input_process=args.input_process) + from GaudiConf.reading import do_unpacking algs = do_unpacking( diff --git a/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_persistreco_mc.qmt b/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_persistreco_mc.qmt index 838b467d1a1ca307dad3ab9f517ebd1d5b05f248..bda056d6f968ef387fdd8126080f3399aaeb6ea1 100644 --- a/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_persistreco_mc.qmt +++ b/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_persistreco_mc.qmt @@ -14,7 +14,7 @@ Test that HLT2 can run from LHCbDirac and that it produces the expected outputs --> - test_prepare_lbexec_hltPASS + test_lbexec_allen_hlt1_pp_mc_2024PASS lbexec 1800