diff --git a/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen.qmt b/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen.qmt index 5cc053d9687f9c35905fa8eda91521412f090d6e..4895b9a277c635632feda888e403b876e77b6acb 100644 --- a/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen.qmt +++ b/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen.qmt @@ -23,8 +23,8 @@ from Moore.qmtest.exclusions import ref_preprocessor remove_output = RegexpReplacer( - when="LHCb__MDFWriter", - orig=r' Output:[^ ]+', + when="MDFWriter", + orig=r' Output: [^ ]+', repl=r'') validateWithReference(preproc=ref_preprocessor + remove_output) diff --git a/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen_Sim10aU1.qmt b/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen_Sim10aU1.qmt index 8b2818d5b32852fa2cee236e3339305a3f12aa32..fa3b297706096c28d69282309318e23163e608ca 100644 --- a/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen_Sim10aU1.qmt +++ b/Hlt/Hlt1Conf/tests/qmtest/dump_for_standalone_allen_Sim10aU1.qmt @@ -24,7 +24,7 @@ from Moore.qmtest.exclusions import ref_preprocessor remove_output = RegexpReplacer( - when="LHCb__MDFWriter", + when="MDFWriter", orig=r' (Output:[^ ]+|Size:.+)', repl=r'') validateWithReference(preproc=ref_preprocessor + remove_output) diff --git a/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/allen_mdf_write.qmt b/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/allen_mdf_write.qmt index 368bfbf609b7e328914eb1c33c60f17f72f3569a..378816c0ea486fdc68e680bd63c914b4c6ce5493 100644 --- a/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/allen_mdf_write.qmt +++ b/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/allen_mdf_write.qmt @@ -72,7 +72,7 @@ if nselected == nread: causes.append('expected at least one event to be filtered out by TwoTrackMVA line') # Check that the filtering works, i.e. that selected events are written out -pattern = re.compile(r'\s+\| "LHCb__MDFWriter_[^"]*"\s+\|\s+(\d+)\s+\|.*') +pattern = re.compile(r'\s+\| "MDFWriter[^"]*"\s+\|\s+(\d+)\s+\|.*') nwritten = -1 for line in stdout.split('\n'): m = re.match(pattern, line) diff --git a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen.ref.armv8.1_a b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen.ref.armv8.1_a index 3d0490f7dc55fe336c97a882a33ed7db2d85c5a4..5113d52386bef100b84e58b3e0bb3596953ae015 100644 --- a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen.ref.armv8.1_a +++ b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen.ref.armv8.1_a @@ -1,4 +1,4 @@ -LHCb__MDFWriter_263a24ae INFO Size:104878064 Output: 'Connection':'/workspace/data/workspace/Moore/build/Hlt/Hlt1Conf/qmtest_tmp/dump/mdf_sim-20231017-vc-mu100/dumped_file.mdf' MD5 sum:b6cb5e4da5985f3693335c4f622f9908 +MDFWriter INFO Size:104878064 Output: 'Connection':'/workspace/data/workspace/Moore/build/Hlt/Hlt1Conf/qmtest_tmp/dump/mdf_sim-20231017-vc-mu100/dumped_file.mdf' MD5 sum:b6cb5e4da5985f3693335c4f622f9908 ApplicationMgr INFO Application Manager Finalized successfully ApplicationMgr INFO Application Manager Terminated successfully DeviceBeamline INFO Number of counters : 1 diff --git a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref index 44d0ed938028bf22cb4b83e36170c9a7176f9f13..3d37fc0acd927817aaac1de6f61ad71ac1641e7b 100644 --- a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref +++ b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref @@ -1,4 +1,4 @@ -LHCb__MDFWriter_5b8f835c INFO +MDFWriter INFO ApplicationMgr INFO Application Manager Finalized successfully ApplicationMgr INFO Application Manager Terminated successfully DeviceBeamline INFO Number of counters : 1 diff --git a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref.detdesc b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref.detdesc index 0f7c4d5a21d4b5954e6fe81562dc6ae3d184d917..b0e5f048860a0c124461734c73c23cecab89f144 100644 --- a/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref.detdesc +++ b/Hlt/Hlt1Conf/tests/refs/dump_for_standalone_allen_Sim10aU1.ref.detdesc @@ -1,4 +1,4 @@ -LHCb__MDFWriter_5b8f835c INFO +MDFWriter INFO ApplicationMgr INFO Application Manager Finalized successfully ApplicationMgr INFO Application Manager Terminated successfully DeviceBeamline INFO Number of counters : 1 diff --git a/Hlt/Hlt2Conf/tests/qmtest/allen_mdf_write_for_hlt2.qmt b/Hlt/Hlt2Conf/tests/qmtest/allen_mdf_write_for_hlt2.qmt index c3311cfffca9482b0f40c1ba44efeaf3e486af6a..f4a28ae08d0b05e77c30ec538054afd81201faea 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/allen_mdf_write_for_hlt2.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/allen_mdf_write_for_hlt2.qmt @@ -72,7 +72,7 @@ if nselected == nread: causes.append('expected at least one event to be filtered out by TwoTrackMVA line') # Check that the filtering works, i.e. that selected events are written out -pattern = re.compile(r'\s+\| "LHCb__MDFWriter_[^"]*"\s+\|\s+(\d+)\s+\|.*') +pattern = re.compile(r'\s+\| "MDFWriter[^"]*"\s+\|\s+(\d+)\s+\|.*') nwritten = -1 for line in stdout.split('\n'): m = re.match(pattern, line) diff --git a/Hlt/Moore/python/Moore/stream_writers.py b/Hlt/Moore/python/Moore/stream_writers.py index 5d725a231d1b42815bbd3d7d4ce8a45b7a80af7b..b9998b144941b2ff75a23e6236c096883a93a6c6 100644 --- a/Hlt/Moore/python/Moore/stream_writers.py +++ b/Hlt/Moore/python/Moore/stream_writers.py @@ -17,7 +17,6 @@ from PyConf import configurable from PyConf.Algorithms import ( AddressKillerAlg, CombineRawBankViewsToRawEvent, - CopyInputStream, RawEventSimpleMover, SelectiveCombineRawBankViewsToRawEvent, ) @@ -26,10 +25,11 @@ from PyConf.application import ( ROOT_KEY, create_or_reuse_rootIOAlg, default_raw_banks, - format_output_location, + input_from_root_file, mdf_writer, online_writer, root_writer, + root_writer_ext, ) from PyConf.components import force_location from PyConf.persistency_locations import ( @@ -69,22 +69,21 @@ sim_veto_list = [ ] microdst_locations = [ - "/Event/HLT2/pSim/MCParticles", - "/Event/HLT2/pSim/MCVertices", - "/Event/MC/Header", - "/Event/MC/OriginalHeader", - "/Event/MC/TrackInfo", + ("/Event/HLT2/pSim/MCParticles", "LHCb::PackedMCParticles"), + ("/Event/HLT2/pSim/MCVertices", "LHCb::PackedMCVertices"), + ("/Event/MC/Header", "LHCb::MCHeader"), + ("/Event/MC/TrackInfo", "LHCb::MCProperty"), ] dst_locations = [ - "/Event/MC/Header", - "/Event/MC/TrackInfo", - "/Event/pSim/MCParticles", - "/Event/pSim/MCVertices", - "/Event/Gen/Header", - "/Event/Gen/BeamParameters", - "/Event/MC/DigiHeader", - "/Event/pSim/Rich/DigitSummaries", + ("/Event/MC/Header", "LHCb::MCHeader"), + ("/Event/MC/TrackInfo", "LHCb::MCProperty"), + ("/Event/pSim/MCParticles", "LHCb::PackedMCParticles"), + ("/Event/pSim/MCVertices", "LHCb::PackedMCVertices"), + ("/Event/Gen/Header", "LHCb::GenHeader"), + ("/Event/Gen/BeamParameters", "LHCb::BeamParameters"), + ("/Event/MC/DigiHeader", "LHCb::ProcessHeader"), + ("/Event/pSim/Rich/DigitSummaries", "LHCb::PackedMCRichDigitSummarys"), ] @@ -146,7 +145,7 @@ def stream_writer( consolidate_views, veto_list = write_spruce( options, stream, hlt_raw_banks, dst_data, dec_reports ) - output_location = consolidate_views.RawEvent + output_location = consolidate_views.RawEvent.location elif process == "pass": consolidate_views, veto_list = write_pass( stream, @@ -155,7 +154,7 @@ def stream_writer( options.input_manifest_file, options.output_manifest_file, ) - output_location = consolidate_views.RawEvent + output_location = consolidate_views.RawEvent.location writers.append(consolidate_views) @@ -174,7 +173,11 @@ def stream_writer( # TODO: Remove one case once lbexec and gaudirun.py agree on a convention. if options.output_type in [MDF_KEY, "RAW"]: - writers.append(mdf_writer(full_fname, consolidate_views.RawEvent)) + writers.append( + mdf_writer( + full_fname, consolidate_views.RawEvent, name=f"MDFWriter_{full_fname}" + ) + ) elif options.output_type == ONLINE: writers.append( online_writer( @@ -202,56 +205,88 @@ def stream_writer( # This part is only meant for simulation # always false for data and by default true for simulation - if options.dstformat == DSTFormatTypes.uDST: + if propagate_mc and options.dstformat == DSTFormatTypes.uDST: # Make the entire datastore anonymous. All persistent info of all addresses is entirely removed. # otherwise, root_writer saves a link to the input event/file # Since the MC particles from HLT2 are not packed into RawEvent, should be declared here writers.append(AddressKillerAlg()) if process == "spruce": - microdst_locations.append("/Event/Spruce/HLT2/pSim/MCParticles") - microdst_locations.append("/Event/Spruce/HLT2/pSim/MCVertices") + microdst_locations.append( + ("/Event/Spruce/HLT2/pSim/MCParticles", "LHCb::PackedMCParticles") + ) + microdst_locations.append( + ("/Event/Spruce/HLT2/pSim/MCVertices", "LHCb::PackedMCVertices") + ) writers.append( - root_writer(full_fname, [output_location] + microdst_locations) + root_writer( + options, + path=full_fname, + locations=[output_location] + + [ + input_from_root_file(path, type).location + for path, type in microdst_locations + ], + name=f"RootOutputStream-{full_fname}", + ) ) - elif options.dstformat == DSTFormatTypes.DST: + elif propagate_mc and options.dstformat == DSTFormatTypes.DST: writers.append(AddressKillerAlg()) # only one version for the locations at current, so use the default one version = _default_persistreco_version - for key, value in digi_to_mc_particles_locations[version].items(): - if key not in ["HcalDigitsV1", "EcalDigitsV1"]: - dst_locations.append("/Event/Link/Raw/" + value) - writers.append(root_writer(full_fname, [output_location] + dst_locations)) - elif write_all_input_leaves and propagate_mc: + other_locations = [ + input_from_root_file( + "/Event/Link/Raw/" + value, "LHCb::LinksByKey" + ).location + for key, value in digi_to_mc_particles_locations[version].items() + if key not in ["HcalDigitsV1", "EcalDigitsV1"] + ] + writers.append( + root_writer( + options, + path=full_fname, + locations=[output_location] + + [ + input_from_root_file(path, type).location + for path, type in dst_locations + ], + optLocations=other_locations, + name=f"RootOutputStream-{full_fname}", + ) + ) + elif propagate_mc and write_all_input_leaves: # Make the entire datastore anonymous. All persistent info of all addresses is entirely removed. # otherwise, root_writer saves a link to the input event/file writers.append(AddressKillerAlg()) locations = [ - format_output_location(l) + getattr(l, "location", l) for l in [output_location] + extra_locations or [] ] - tes_veto_locations = [ - format_output_location(l, add_depth=False) for l in veto_list or [] - ] writers.append( - CopyInputStream( - name="CopyInputStream", - InputFileLeavesLocation=create_or_reuse_rootIOAlg( + root_writer_ext( + options, + path=full_fname, + inputLeavesLocations=create_or_reuse_rootIOAlg( options ).InputLeavesLocation, - OptItemList=locations, - TESVetoList=tes_veto_locations, - Output="DATAFILE='{}' SVC='Gaudi::RootCnvSvc' OPT='RECREATE'".format( - full_fname - ), + optLocations=locations, + vetoList=veto_list, + name=f"RootOutputStream-{full_fname}", ) ) else: # Make the entire datastore anonymous. All persistent info of all addresses is entirely removed. # otherwise, root_writer saves a link to the input event/file writers.append(AddressKillerAlg()) - writers.append(root_writer(full_fname, [output_location])) + writers.append( + root_writer( + options, + path=full_fname, + optLocations=[output_location], + name=f"RootOutputStream-{full_fname}", + ) + ) else: raise NotImplementedError() diff --git a/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_reco_only_mc.qmt b/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_reco_only_mc.qmt index 05b9004ae7592f4cea3755134c963d8368658a8b..e3a4b51b4f6fc02ff09d6571a29a07b86fed3a90 100644 --- a/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_reco_only_mc.qmt +++ b/Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_flagging_reco_only_mc.qmt @@ -20,6 +20,7 @@ Test that HLT2 can run from LHCbDirac and that it produces the expected outputs test_prepare_lbexec_hltPASS + test_lbexec_allen_hlt1_pp_mc_2024PASS lbexec 1800 diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py index 96cd62d4f2d97f036fcc186f6db7330e7292ac95..87e403fdd8aaa4c58c044c7716cee88cfa86a566 100755 --- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py +++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py @@ -40,7 +40,7 @@ def combiner_digi(): data = [ spkiller, raw_event_combiner, - root_writer(options.output_file, [rawEventLocation.location]), + root_writer(options, locations=[rawEventLocation.location]), ] return Reconstruction("write_digi", data)