diff --git a/Hlt/Hlt2Conf/options/hlt2_check_output.py b/Hlt/Hlt2Conf/options/hlt2_check_output.py index 6a6ca8d22f6b6b3cf23b8cae74cbbd6aee782036..703bd4b1421514507136111526858062b1b2ebd1 100644 --- a/Hlt/Hlt2Conf/options/hlt2_check_output.py +++ b/Hlt/Hlt2Conf/options/hlt2_check_output.py @@ -46,7 +46,7 @@ CondDB(Upgrade=True) # Disable warning about not being able to navigate ancestors IODataManager(DisablePFNWarning=True) manifest = load_manifest(sys.argv[2]) -algs = do_unpacking(manifest, process='Hlt2', output_level=4) +algs = do_unpacking(manifest, process='Hlt2') appmgr = ApplicationMgr(TopAlg=algs) appmgr.ExtSvc += [configured_ann_svc()] diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines.py b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines.py index 93e18bc79317dfcee4351af0ac978c0fc81923d2..c1bcd2fdf93b40b8ea7a30e9134ce50452aff250 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines.py @@ -106,7 +106,5 @@ public_tools = [ options.scheduler_legacy_mode = False -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_streams, public_tools, analytics=True) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py index aae1a8308077ea0f59426e8562bcce2fe4ce6fea..c24e933c00f912ddbbeaf230b151ad4903d6c25b 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py @@ -115,6 +115,6 @@ Moore.streams_spruce.stream_banks = { public_tools = [stateProvider_with_simplified_geom()] with reconstruction.bind( - from_file=True, spruce=True, manifest_file=options.input_manifest_file -), CombineRawBankViewsToRawEvent.bind(OutputLevel=4): + from_file=True, + spruce=True), CombineRawBankViewsToRawEvent.bind(OutputLevel=4): config = run_moore(options, make_streams, public_tools, analytics=True) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime.py b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime.py index f6b21aa966f3d76abfa49aec337ff65957193ab9..28d3bc6bd9f8a9a977006eda4380ae4a6285a9a1 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime.py @@ -34,6 +34,7 @@ options.input_files = [ 'mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/rta/samples/mc/Hlt1Hlt2filtered_MinBias_sprucing/hlt2_2or3bodytopo_realtime_newPacking_newDst.mdf' ] options.input_manifest_file = 'root://eoslhcb.cern.ch//eos/lhcb/wg/rta/samples/mc/Hlt1Hlt2filtered_MinBias_sprucing/hlt2_2or3bodytopo_realtime_newPacking_newDst.tck.json' + metainfo_repos.global_bind(extra_central_tags=['key-5b3d0522']) options.evt_max = -1 @@ -57,7 +58,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime_test_old_json.py b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime_test_old_json.py index 625b4c02a0abfb21d896954d333283fb00e127d8..3ace733182119b2989addd27c9244468c1b919f6 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime_test_old_json.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_realtime_test_old_json.py @@ -51,7 +51,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py index 077670f45f2216e1ff486c01a286b147ef24d32a..ed8795b2fab5ff95260c23f1725be46c5c1b8df1 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py @@ -49,7 +49,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime.py index e33bfe845b46b3a2083f96da6eccf591197bc985..60d7b00ebbce2b44493f08fd8a8c4e0d215990cd 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime.py @@ -41,7 +41,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_dstinput.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_dstinput.py index a4b16217e03bb76ffb5532a1ff9fb2bb39f642df..8f15d0a3d38df42cb47fd629aaf8e00c3e3d2ffa 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_dstinput.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_dstinput.py @@ -45,7 +45,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_extraoutputs.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_extraoutputs.py index eb7e27b16159a8f398409c9103cfa285be94c28c..fb9341467d69d7b754e01f4ff1df7c4e969b6a07 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_extraoutputs.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_extraoutputs.py @@ -41,7 +41,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_persistreco.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_persistreco.py index b7a659ce77f2b369eb9730144f3869f981349e24..e347b0b2854b3dfdffa8b7c035a7b716017adaca 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_persistreco.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_realtime_persistreco.py @@ -41,7 +41,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_hlt2filter.py b/Hlt/Hlt2Conf/options/sprucing/spruce_hlt2filter.py index 1415f674cb5ddde0b29943e4ba4e4abc2410e9e7..aefd8a8d0bf63ddd41761dc7d0671249dbb5ef99 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_hlt2filter.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_hlt2filter.py @@ -64,7 +64,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough.py b/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough.py index 0c01ac2853655f3e83f496ef9f96853267259af6..d0dd652f9017e6fd766497f9414282715bbc91ee 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough.py @@ -51,7 +51,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough_dstinput.py b/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough_dstinput.py index e6eef9269a794835a30e1651e4187f0a4a21c9cf..a5d2cbe5a52c8a1d5e140b07f080166a12fb2856 100644 --- a/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough_dstinput.py +++ b/Hlt/Hlt2Conf/options/sprucing/spruce_passthrough_dstinput.py @@ -53,7 +53,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/options/streaming/spruce_test_streaming.py b/Hlt/Hlt2Conf/options/streaming/spruce_test_streaming.py index 96ed793e66da4345bb8c4673cbb6fd28ef32e53e..b6fb82bd9067fa520285b680b2b92b3dbc03cacc 100644 --- a/Hlt/Hlt2Conf/options/streaming/spruce_test_streaming.py +++ b/Hlt/Hlt2Conf/options/streaming/spruce_test_streaming.py @@ -68,7 +68,5 @@ pprint(Moore.streams_spruce.stream_banks) public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_streams, public_tools) diff --git a/Hlt/Hlt2Conf/tests/options/hlt2_passthrough_persistreco_check.py b/Hlt/Hlt2Conf/tests/options/hlt2_passthrough_persistreco_check.py index 920dcdb73e17298ac499e85f299a4b52a1dfcdae..b22f3fea1432e3c71e1fbba135827e30f5b1b531 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt2_passthrough_persistreco_check.py +++ b/Hlt/Hlt2Conf/tests/options/hlt2_passthrough_persistreco_check.py @@ -29,19 +29,17 @@ from GaudiConf import IOExtension from Configurables import (ApplicationMgr, CondDB, LHCbApp, IODataManager, HistogramPersistencySvc) from GaudiConf.reading import load_manifest -from GaudiConf.PersistRecoConf import PersistRecoPacking from GaudiConf.reading import do_unpacking from PyConf.application import configured_ann_svc from Hlt2Conf.check_output import check_persistreco +from PyConf.packing import default_persisted_locations parser = argparse.ArgumentParser() parser.add_argument('input', help='Input filename') parser.add_argument('hlt2_tck', help='HLT2 JSON TCK dump') args = parser.parse_args() -prp = PersistRecoPacking(stream='/Event/HLT2/', data_type='Upgrade') - algs = do_unpacking(load_manifest(args.hlt2_tck), process='Hlt2') ##Prepare application @@ -74,7 +72,8 @@ while True: if not TES['/Event']: break - check_persistreco(TES, prp.unpackedLocations(), "Turbo", N_TURBO) + locations = default_persisted_locations(stream="/Event/HLT2") + check_persistreco(TES, locations.values(), "Turbo", N_TURBO) hlt2reports = TES['/Event/Hlt2/DecReports'] hlt2report = hlt2reports.decReport( diff --git a/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check.py b/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check.py index a8692ff7b5153cbdd0731ef5c1d15dfd19570d69..145202395be92f8669df743adab1ec1ca83ff9c3 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check.py +++ b/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check.py @@ -34,7 +34,7 @@ from Configurables import ( HistogramPersistencySvc, ) -from GaudiConf.PersistRecoConf import PersistRecoPacking +from PyConf.packing import default_persisted_locations from GaudiConf.reading import do_unpacking from Hlt2Conf.check_output import check_persistreco from GaudiConf.reading import load_manifest @@ -61,10 +61,6 @@ def advance_HLT(decision): return -from Moore.persistence import DEFAULT_OUTPUT_PREFIX -prp = PersistRecoPacking( - stream=DEFAULT_OUTPUT_PREFIX, reco_stream="HLT2", data_type='Upgrade') - cfg = load_manifest(args.hlt2_manifest) ##Prepare application @@ -76,7 +72,7 @@ LHCbApp( ) CondDB(Upgrade=True) -algs = do_unpacking(cfg, process='Hlt2', output_level=4) +algs = do_unpacking(cfg, process='Hlt2') app = ApplicationMgr(TopAlg=algs) app.ExtSvc += [configured_ann_svc()] @@ -102,7 +98,8 @@ for ii in range(nevents): if ii == 0: TES.dump() - check_persistreco(TES, prp.unpackedLocations(), "Hlt2") + locations = default_persisted_locations(stream="/Event/HLT2") + check_persistreco(TES, locations.values(), "Hlt2") # We should still be persisting the HLT2 line candidates if len(TES["/Event/HLT2/" + line + "/Particles"]) < 1: @@ -122,7 +119,8 @@ for ii in range(nevents): if ii == 0: TES.dump() - check_persistreco(TES, prp.unpackedLocations(), "Hlt2") + locations = default_persisted_locations(stream="/Event/HLT2") + check_persistreco(TES, locations.values(), "Hlt2") # We should still be persisting the HLT2 line candidates if len(TES["/Event/HLT2/" + line + "/Particles"]) < 1: diff --git a/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check_flavourtags.py b/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check_flavourtags.py index 2913e23d6d154193f72fa1fe07025b4509899e6b..4cac596a8a550c42dc70329e5551f32e45eefeb1 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check_flavourtags.py +++ b/Hlt/Hlt2Conf/tests/options/hlt2_persistreco_check_flavourtags.py @@ -45,7 +45,7 @@ process = "Hlt2" cfg = load_manifest(args.hlt2_manifest) -algs = do_unpacking(cfg, process=process, output_level=4) +algs = do_unpacking(cfg, process=process) app = ApplicationMgr(TopAlg=algs) app.ExtSvc += [configured_ann_svc()] diff --git a/Hlt/Hlt2Conf/tests/options/hlt_filters_test_sprucepass.py b/Hlt/Hlt2Conf/tests/options/hlt_filters_test_sprucepass.py index 0dde20d357f8aa8e1e8f5a46e21516d9de029368..255c0842ae546ccb9321b4c4d5e8f1ba88f15c87 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt_filters_test_sprucepass.py +++ b/Hlt/Hlt2Conf/tests/options/hlt_filters_test_sprucepass.py @@ -72,7 +72,5 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind( - from_file=True, spruce=True, - manifest_file=options.input_manifest_file): +with reconstruction.bind(from_file=True, spruce=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py index bee5e52d621909cf338d617a7f97e4c99f4c85a7..0e3eeae9932bf512ce55da858f9a83e2abd95daa 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py @@ -23,7 +23,7 @@ from Configurables import ( HistogramPersistencySvc, ) -from GaudiConf.PersistRecoConf import PersistRecoPacking +from PyConf.packing import default_persisted_locations from GaudiConf.reading import do_unpacking @@ -61,25 +61,19 @@ process = args.p stream = args.s if process == "Spruce": - FULL_ROOT = "/Event/Spruce" RECO_ROOT = "/Event/Spruce/HLT2" loc = "Spruce" dec_to_check = "Spruce_Test_line" elif process == "Turbo": - FULL_ROOT = "/Event/HLT2" RECO_ROOT = "/Event/HLT2" loc = "HLT2" dec_to_check = "PassThrough" -prp = PersistRecoPacking( - stream=FULL_ROOT, reco_stream='HLT2', data_type='Upgrade') - algs = do_unpacking( load_manifest(args.t), process=process, stream=stream, - output_level=4, simulation=True, raw_event_format=0.3) @@ -202,7 +196,8 @@ for ii in range(nevents): print("MC ERROR MC vertices not correctly propagated") # Forth step: check persistency of packed containers - check_persistreco(TES, prp.unpackedLocations(), "Spruce") + locations = default_persisted_locations(stream=RECO_ROOT) + check_persistreco(TES, locations.values(), "Spruce") # Check a random RawBank is populated bank_loc = '/Event/' + stream diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_extraoutputs.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_extraoutputs.py index 12eb2ad0eb4e2c32b5ecd25343521dd507ef3779..6403d6623ad3213b24f6fbc745471ac4f22ec959 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_extraoutputs.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_extraoutputs.py @@ -32,13 +32,11 @@ from Configurables import ( ) from PyConf.application import configured_ann_svc -from GaudiConf.PersistRecoConf import PersistRecoPacking +from PyConf.packing import default_persisted_locations from GaudiConf.reading import do_unpacking from GaudiConf.reading import load_manifest -from Gaudi.Configuration import WARNING as OUTPUTLEVEL - from Hlt2Conf.check_output import check_persistreco @@ -66,14 +64,8 @@ FULL_ROOT = "/Event/Spruce" loc = "Spruce" dec_to_check = "Spruce_Test_line_extraoutputs" -prp = PersistRecoPacking( - stream=FULL_ROOT, reco_stream='HLT2', data_type='Upgrade') - algs = do_unpacking( - load_manifest(args.t), - process=process, - output_level=OUTPUTLEVEL, - raw_event_format=0.3) + load_manifest(args.t), process=process, raw_event_format=0.3) app = ApplicationMgr(TopAlg=algs) app.ExtSvc += [configured_ann_svc()] @@ -166,7 +158,8 @@ for ii in range(nevents): print("Persistency ERROR extra_outputs not being saved correctly.") # Forth step: check persistency of packed containers - check_persistreco(TES, prp.unpackedLocations(), "Spruce") + locations = default_persisted_locations(stream="/Event/Spruce/HLT2") + check_persistreco(TES, locations.values(), "Spruce") # Check a random RawBank is populated bank_loc = '/Event/default' diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_persistreco.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_persistreco.py index 4dc94cca892fa9c5fe1c8e16a9cc4188ad3babf2..771e6a3e34f94b9a6ee590ae73b6f2ba185b0f46 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_persistreco.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check_persistreco.py @@ -31,7 +31,7 @@ from Configurables import ( HistogramPersistencySvc, ) -from GaudiConf.PersistRecoConf import PersistRecoPacking +from PyConf.packing import default_persisted_locations from GaudiConf.reading import do_unpacking from PyConf.application import configured_ann_svc from GaudiConf.reading import load_manifest @@ -63,14 +63,8 @@ FULL_ROOT = "/Event/Spruce" loc = "Spruce" dec_to_check = "Spruce_Test_line_persistreco" -prp = PersistRecoPacking( - stream="/Event/Spruce", reco_stream="HLT2", data_type='Upgrade') - algs = do_unpacking( - load_manifest(args.t), - process=process, - output_level=4, - raw_event_format=0.3) + load_manifest(args.t), process=process, raw_event_format=0.3) appmgr = ApplicationMgr(TopAlg=algs) appmgr.ExtSvc += [configured_ann_svc()] @@ -167,7 +161,8 @@ for ii in range(nevents): "persistency ERROR persistreco objects not being saved correctly.") # Forth step: check persistency of packed containers - check_persistreco(TES, prp.unpackedLocations(), "Spruce") + locations = default_persisted_locations(stream="/Event/Spruce/HLT2") + check_persistreco(TES, locations.values(), "Spruce") # Check a random RawBank is populated bank_loc = '/Event/default' diff --git a/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py b/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py index 62080ac3e838f1c591a871daec61397dd41ba13b..c33e7986d885c0c19d7e8deb91e107ecce90d143 100644 --- a/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py +++ b/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py @@ -33,7 +33,6 @@ from Configurables import ( HistogramPersistencySvc, ) from Moore.persistence import DEFAULT_OUTPUT_PREFIX -from GaudiConf.PersistRecoConf import PersistRecoPacking from GaudiConf.reading import do_unpacking from PyConf.application import configured_ann_svc from GaudiConf.reading import load_manifest @@ -88,10 +87,7 @@ elif args.process == "Spruce": else: print("configuration ERROR process not supported") -algs = do_unpacking( - cfg, process=args.process, stream=args.stream, output_level=4) -prpacking = PersistRecoPacking( - stream=TES_ROOT, reco_stream='HLT2', data_type='Upgrade') +algs = do_unpacking(cfg, process=args.process, stream=args.stream) mgr = ApplicationMgr(TopAlg=algs) mgr.ExtSvc += [configured_ann_svc()] diff --git a/Hlt/Hlt2Conf/tests/qmtest/hlt2_combinations_particle_v2.qmt b/Hlt/Hlt2Conf/tests/qmtest/hlt2_combinations_particle_v2.qmt index acdeca4ea8688943293ecc26870db04f8bd2c222..bb7ae12bfcccecb6d37a7cfea040ae15e19a84cd 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/hlt2_combinations_particle_v2.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/hlt2_combinations_particle_v2.qmt @@ -27,9 +27,8 @@ from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, stdout=remove_known_warnings(stdout)) -from Moore.qmtest.exclusions import skip_initialize, remove_known_fluctuating_counters, preprocessor as moore_preprocessor -from RecConf.QMTest.exclusions import preprocessor as RecPreprocessor -validateWithReference(preproc = skip_initialize + moore_preprocessor + RecPreprocessor, counter_preproc = remove_known_fluctuating_counters) +from Moore.qmtest.exclusions import ref_preprocessor, remove_known_fluctuating_counters +validateWithReference(preproc=ref_preprocessor, counter_preproc = remove_known_fluctuating_counters) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_standard_particles.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_standard_particles.qmt index 49d2c549cebb499df80e698dc16fb6f5098de229..772f6a58952ad7f6c517fde17d487e9ab7e0ab17 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_standard_particles.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_standard_particles.qmt @@ -23,6 +23,8 @@ HiveDataBrokerSvc().OutputLevel = 5 true -countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 2}) +from Moore.qmtest.exclusions import remove_known_warnings +countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, + stdout=remove_known_warnings(stdout)) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 93e2f63b3e9de77389c53f03b88ddca9a6331917..0800b171e34ad2c7b2e3c4f13ba48f9893db1070 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -9,6 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from __future__ import absolute_import +from shutil import copy import re, logging, inspect, itertools from collections import namedtuple from PyConf import configurable @@ -31,6 +32,7 @@ from PyConf.application import ( root_writer, make_data_with_FetchDataFromFile, ) + from PyConf.filecontent_metadata import generate_encoding_dictionary, register_encoding_dictionary from GaudiConf.reading import unpack_rawevent, mc_unpackers from PyConf.utilities import ConfigurationError @@ -114,6 +116,7 @@ class Reconstruction(namedtuple('Reconstruction', ['node'])): # noqa def report_writers_nodes(streams, data_type, process, + output_manifest_file, associate_mc=False, clone_mc=False): """Return the control flow node and locations to persist of the default reports writers. @@ -207,10 +210,16 @@ def report_writers_nodes(streams, algs.append(srw) new_hlt_banks['HltSelReports'] = srw.RawEvent elif process == "hlt2": - (line_output_cf, - line_output_locations, packed_data) = persist_line_outputs( - physics_lines, data_type, erw.DecReportsLocation, associate_mc, - process.capitalize()) + (line_output_cf, line_output_locations, + packed_data) = persist_line_outputs( + physics_lines, + data_type, + erw.DecReportsLocation, + associate_mc, + process.capitalize(), + output_manifest_file, + clone_mc=options.simulation and options.input_type == ROOT_KEY) + algs.append(line_output_cf) new_hlt_banks['DstData'] = packed_data.OutputRawEvent extra_locations_to_persist.extend(line_output_locations) @@ -238,6 +247,7 @@ def report_writers_nodes(streams, erw.DecReportsLocation, associate_mc, process.capitalize(), + output_manifest_file, stream="/Event/Spruce", reco_stream="/Event/HLT2", clone_mc=options.simulation and options.input_type == ROOT_KEY) @@ -246,6 +256,13 @@ def report_writers_nodes(streams, algs.append(line_output_cf) extra_locations_to_persist.extend(line_output_locations) + if process == "pass": + if not options.input_manifest_file: + raise RuntimeError( + ' pass-through configuration -- must specify an input manifest' + ) + copy(options.input_manifest_file, options.output_manifest_file) + node = CompositeNode( 'report_writers', combine_logic=NodeLogic.NONLAZY_OR, @@ -449,6 +466,7 @@ def moore_control_flow(options, streams, process, allen_hlt1, analytics=False): streams, options.data_type, process, + options.output_manifest_file, # Can only run association when we have access to the linker tables. Only want association for hlt step associate_mc=process == "hlt2" and options.simulation and options.input_type == ROOT_KEY and options.output_type == ROOT_KEY, @@ -465,8 +483,7 @@ def moore_control_flow(options, streams, process, allen_hlt1, analytics=False): stream_writers_setup = [] unpack = [] - if (options.output_file or options.output_type == ONLINE) and ( - process == "hlt1" or process == "hlt2") and not analytics: + if (process == "hlt1" or process == "hlt2") and not analytics: for stream, stream_lines in streams.items(): rbw = Hlt__RoutingBitsWriter( RoutingBits=get_default_routing_bits({ @@ -492,9 +509,18 @@ def moore_control_flow(options, streams, process, allen_hlt1, analytics=False): bank_types = list(HLT1_REPORT_RAW_BANK_TYPES | HLT2_REPORT_RAW_BANK_TYPES | DETECTOR_RAW_BANK_TYPES) + + if process == "spruce": + # sprucing already unpacked the DstData in reco_objects_for_spruce + bank_types_to_unpack = [ + b for b in bank_types if b != 'DstData' + ] + else: + bank_types_to_unpack = bank_types unpackrawevent = unpack_rawevent( - bank_types=bank_types, configurables=False) + bank_types=bank_types_to_unpack, configurables=False) unpack.append(unpackrawevent) + ## Hack to make `extra_locations_to_persist` objects writable in pass through if options.simulation and options.input_type == 'ROOT' and process == "pass": unpack += mc_unpackers(configurables=False) @@ -618,7 +644,7 @@ def run_moore(options, public_tools (list): list of public `Tool` instances to configure allen_hlt1: Configure Allen Hlt1 lines when processing Hlt2 on Allen filtered Hlt1 sample, Moore Hlt1 lines are confiugred when allen_hlt1=False analytics (bool, optional): For use only in rate/event size analysis. Defaults to False. - exclude_incompatible (bool, optional): Exclude the lines that are incompatible with multithreaded mode. Defaults to True. + exclude_incompatible (bool, optional): Exclude the lines that are incompatible with multithreaded mode. Defaults to True. """ # First call configure_input for its important side effect of # changing the default values of default_raw_event's arguments. diff --git a/Hlt/Moore/python/Moore/persistence/__init__.py b/Hlt/Moore/python/Moore/persistence/__init__.py index a57514811d06cbb9445b9072d70b68de1308ee3c..02d8e011311d26fdf981c818f2bf20e113bb1b3c 100644 --- a/Hlt/Moore/python/Moore/persistence/__init__.py +++ b/Hlt/Moore/python/Moore/persistence/__init__.py @@ -11,23 +11,22 @@ """Configuration for persisting HLT2 objects in output ROOT/MDF files.""" from __future__ import absolute_import import itertools -import logging, os +import logging, os, json from pprint import pformat from Configurables import HltLinePersistenceSvc +#from RecoConf.data_from_file import unpacked_mc_locations from PyConf import configurable from PyConf.control_flow import CompositeNode, NodeLogic from PyConf.components import get_output -from PyConf.location_prefix import prefix, unpacked_prefix, packed_prefix +from PyConf.location_prefix import prefix from PyConf.filecontent_metadata import register_encoding_dictionary from GaudiConf.reading import type_map -from GaudiConf.PersistRecoConf import PersistRecoPacking from .cloning import clone_line_outputs -from .packing import pack_stream_objects, pack_stream_mc, pack_stream_mc_locations -from .persistreco import persistreco_line_outputs, persistreco_line_outputs_packed -from .serialisation import serialise_packed_containers +from .packing import pack_stream_objects, pack_stream_mc, pack_stream_mc_locations, packers_map +from .persistreco import persistreco_line_outputs from .truth_matching import truth_match_lines, CHARGED_PP2MC_LOC, NEUTRAL_PP2MC_LOC log = logging.getLogger(__name__) @@ -103,6 +102,43 @@ def get_type(dh): return None +def get_packed_locations(lines, inputs, stream): + + line_locs = set() + + for l in lines: + for dh in l.objects_to_persist: + line_locs.add(dh) + + if "PP2MCPRelations" in inputs.keys(): + for loc in inputs["PP2MCPRelations"]: + line_locs.add(loc) + packed_dhs = [] + + types = type_map() + k = list(types.keys()) + v = list(types.values()) + + prdict = persistreco_line_outputs() + + for key, locs in inputs.items(): + for i in locs: + if i in line_locs or i in prdict.values(): + if isinstance(i, str): + t = k[v.index(key)] + + packed_dhs += [(prefix(i, stream), t)] + else: + t = i.type + if i.type == "unknown_t": + t = k[v.index(key)] + + packed_dhs += [(prefix(i.location, stream), t)] + + packed_dhs = list(dict.fromkeys(packed_dhs)) + return {'PackedLocations': packed_dhs} + + @configurable def persist_line_outputs( lines, @@ -110,9 +146,10 @@ def persist_line_outputs( dec_reports, associate_mc, source_id, + output_manifest_file, stream=DEFAULT_OUTPUT_PREFIX, #this is where everything goes reco_stream=DEFAULT_OUTPUT_PREFIX, #this is where reco objects come from - clone_mc=True): + clone_mc=False): """Return CF node and output locations of the HLT2 line persistence. Returns: @@ -148,13 +185,15 @@ def persist_line_outputs( l.decision_name: [dh.location for dh in l.objects_to_persist] for l in lines }, - TurboPPLines=[l.decision_name for l in lines if l.persistreco], + #TurboPPLines=[l.decision_name for l in lines if l.persistreco], + TurboPPLines=[], ) if log.isEnabledFor(logging.DEBUG): log.debug('line_locations: ' + pformat(persistence_svc.Locations)) - # Make a dictionary for all known object types with empty values - inputs = PersistRecoPacking().dictionary() + # Make a dictinary for all known object types with emty values + p_map = packers_map() + inputs = {t: [] for t in p_map.keys()} #add line outputs to fill the dictionary inputs = _referenced_inputs(lines, inputs) @@ -174,6 +213,14 @@ def persist_line_outputs( for p in protoparticle_relations: inputs["PP2MCPRelations"] += [p] + if output_manifest_file: + with open(output_manifest_file, 'w') as f: + json.dump( + get_packed_locations(lines, inputs, stream), + f, + indent=4, + sort_keys=True) + locify = lambda i: i.location if hasattr(i, 'location') else i inputs = {t: [locify(i) for i in dhs] for t, dhs in inputs.items()} @@ -200,30 +247,8 @@ def persist_line_outputs( pformat(output_cloner_locations)) cf.append(output_cloner_cf) - #Make a dictionary for output packer locations - #For line outputs, "stream+/p" added to input locations - #For reco objects, there are pre-defined output locations - #This is to be able to find reco objects regardless of their producer - prdict_packed = persistreco_line_outputs_packed(stream, reco_stream) - outputs = {} - for key, value in inputs.items(): - outputs[key] = [] - for v in value: - if v in prdict_packed.keys(): - outputs[key] += [prdict_packed[v]] #reco - else: - outputs[key] += [packed_prefix(v, stream)] #line - - prpacking = PersistRecoPacking( - stream=stream, - unpacked=inputs, - packed=outputs, - data_type=data_type, - ) - ### TODO: reduce the set of encoding keys to the smallest possible one... - locations = set([ unpacked_prefix(i, stream) for i in prpacking.packedLocations() ]) | \ - set([ i for i in prpacking.unpackedLocations()]) | \ + locations = set([ i for ilist in inputs.values() for i in ilist]) | \ set([ i.location for i in itertools.chain( *_referenced_locations(lines).values()) ]) if clone_mc: @@ -235,26 +260,22 @@ def persist_line_outputs( register_encoding_dictionary("PackedObjectLocations", sorted(locations)), 16) - packer_cf, packer_handles = pack_stream_objects(stream, prpacking, - encoding_key) - cf.append(packer_cf) + packers_cf, serialisation_cf, output_raw_data = pack_stream_objects( + stream, inputs, encoding_key, source_id) - packer_mc_locations = [] + cf.append(packers_cf) + packer_mc_locations = [] if clone_mc: mc_packer_cf, packer_mc_locations = pack_stream_mc(prefix(mc_stream)) cf.append(mc_packer_cf) - if log.isEnabledFor(logging.DEBUG): - log.debug('packer_handles: ' + pformat(packer_handles)) - log.debug('packer_mc_locations: ' + pformat(packer_mc_locations)) - - serialisation_cf, output_raw_data = serialise_packed_containers( - packer_handles, source_id) + cf.append(serialisation_cf) if log.isEnabledFor(logging.DEBUG): + log.debug('packer_locations: ' + pformat(inputs.values())) + log.debug('packer_mc_locations: ' + pformat(packer_mc_locations)) log.debug('output_raw_data: %s', pformat(output_raw_data)) - cf.append(serialisation_cf) control_flow_node = CompositeNode( "hlt2_line_output_persistence", diff --git a/Hlt/Moore/python/Moore/persistence/cloning.py b/Hlt/Moore/python/Moore/persistence/cloning.py index a1f869f219f15b12e99f590b7300eb22d576ee2a..60ab8b502d65b423f8a8c57d8d43fea4e6d2ac72 100644 --- a/Hlt/Moore/python/Moore/persistence/cloning.py +++ b/Hlt/Moore/python/Moore/persistence/cloning.py @@ -144,6 +144,7 @@ def clone_line_outputs(persistence_svc, CloneClustersAlways=False, CloneDigitsAlways=False, ) + else: # Always clone all information associated to CALO objects # This may end up being impossible due to bandwidth constraints, but @@ -187,11 +188,12 @@ def clone_line_outputs(persistence_svc, ICloneCaloCluster=cluster_cloner, ICloneProtoParticle=protoparticle_cloner, ICloneParticle=particle_cloner, + #TurboPPICloneVertexBase='VertexBaseFromRecVertexClonerNoTracks', # We need to declare outputs since packers are now using DataHandles # and CopyLinePersistenceLocations is not. To be removed when this # algorithm is dropped or ported to DataHandles. ExtraOutputs=line_outputs, - OutputLevel=4, + OutputLevel=5, ) p2pv_cloner = CopyParticle2PVRelationsFromLinePersistenceLocations( diff --git a/Hlt/Moore/python/Moore/persistence/packing.py b/Hlt/Moore/python/Moore/persistence/packing.py index 2d55d78a9971d805884453235099d86cc0bb42e9..a8024185d065c7ac04a41e78f14cc62fdebe05a8 100644 --- a/Hlt/Moore/python/Moore/persistence/packing.py +++ b/Hlt/Moore/python/Moore/persistence/packing.py @@ -11,14 +11,11 @@ import logging import os -from PyConf.Algorithms import ( - PackMCParticle, - PackMCVertex, -) -from PyConf.components import force_location +from PyConf.packing import packers_map +from PyConf.components import get_output, force_location from PyConf.control_flow import CompositeNode, NodeLogic -from Gaudi.Configuration import WARNING as OUTPUTLEVEL +from PyConf.Algorithms import PackMCParticle, PackMCVertex, HltPackedBufferWriter log = logging.getLogger(__name__) @@ -26,32 +23,53 @@ from PyConf import configurable @configurable -def pack_stream_objects(stream, prpacking, encoding_key, enable_check=False): - """Return a list of packers that will produce all packed output. - +def pack_stream_objects(stream, + inputs, + encoding_key, + source_id, + enable_check=False): + """Return CF node that packs and serialises a set of containers to a raw bank. Args: - stream (str): TES root containing objects to be packed. - prpacking (PersistRecoPacking object): PersistRecoPacking object that describes packing configuration. + stream (str): TES root containing objects to be persisted. + inputs (map): Type: locations to be persisted Returns: - algs (list): Algorithms to run the packing. - outputs (list of handles): data handles that should be persisted, in the - specification used by ROOT output writers (e.g. OutputStream). + serialisation_node (CompositeNode). + output_raw_data (DataHandle): Raw event with serialised data, + i.e. the DstData bank. """ + p_map = packers_map() + packers = [] + + for t, p in p_map.items(): + if t in inputs.keys(): + packer = p( + InputName=[force_location(loc) for loc in inputs[t]], + EnableCheck=enable_check, + EncodingKey=encoding_key) - persistreco_packers = prpacking.packers( - output_level=OUTPUTLEVEL, - enable_check=enable_check, - encoding_key=encoding_key) + packers += [packer] packers_cf = CompositeNode( "packers", - children=persistreco_packers, + children=packers, combine_logic=NodeLogic.NONLAZY_OR, force_order=True, ) - return packers_cf, [p.outputs["OutputName"] for p in persistreco_packers] + packers_output_locations = [ + get_output(p.outputs["OutputName"]).location for p in packers + ] + + bank_writer = HltPackedBufferWriter( + PackedContainers=packers_output_locations, SourceID=source_id) + + serialisation_cf = CompositeNode( + "serialisation", + children=[bank_writer], + ) + + return packers_cf, serialisation_cf, bank_writer def pack_stream_mc_locations(stream): diff --git a/Hlt/Moore/python/Moore/persistence/persistreco.py b/Hlt/Moore/python/Moore/persistence/persistreco.py index 0a0a2e9edfa8ee7aba01269be4cdc96f0c577b58..27acf78f6efcce953cacdea620ce03fe692de655 100644 --- a/Hlt/Moore/python/Moore/persistence/persistreco.py +++ b/Hlt/Moore/python/Moore/persistence/persistreco.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # @@ -18,9 +18,8 @@ Some objects which are anyhow persisted as part of the usual line output, such as primary vertices, are treated different if the line has PersistReco enabled. See the cloning configuration for those differences. """ -from PyConf.components import get_output -from PyConf.location_prefix import prefix, packed_prefix -from GaudiConf.PersistRecoConf import default_persisted_locations + +from PyConf.packing import default_persisted_locations from RecoConf.reconstruction_objects import reconstruction @@ -37,20 +36,3 @@ def persistreco_line_outputs(): if key in to_be_persisted: prdict[key] = val return prdict - - -def persistreco_line_outputs_packed(stream, reco_stream): - """Return a dict of data handles that define reconstruction to be persisted.""" - - stream_loc = { - key: prefix(value, reco_stream) - for key, value in default_persisted_locations().items() - } - - prdict = persistreco_line_outputs() - packed = { - prefix(get_output(val).location, stream): packed_prefix( - stream_loc[key], stream) - for key, val in prdict.items() - } - return packed diff --git a/Hlt/RecoConf/options/run_two_hlt2_recos.py b/Hlt/RecoConf/options/run_two_hlt2_recos.py index f568a2b9ec548f2af849965778e1c4ed42270c1e..9533fbafd4d49e6de29f2433ab3d1b2ad51f8437 100644 --- a/Hlt/RecoConf/options/run_two_hlt2_recos.py +++ b/Hlt/RecoConf/options/run_two_hlt2_recos.py @@ -12,7 +12,7 @@ from Moore import options, run_reconstruction from Moore.config import Reconstruction from RecoConf.hlt2_global_reco import reconstruction from PyConf.Algorithms import PrForwardTrackingVelo -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location def make_two_reconstructions(): diff --git a/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py b/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py index 4c06fe704bb6ac2ac5819f4d13c43afa3e0fa724..2ba18351f56f032c346b4cc3f43b1a9684fe94b7 100644 --- a/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py +++ b/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from PyConf import configurable -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.Algorithms import ( AcceptanceBremAlg, AcceptanceEcalAlg, AcceptanceHcalAlg, TrackToEcalEnergyAlg, TrackToHcalEnergyAlg, CaloChargedPIDsAlg, diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py b/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py index 6664bac6e6fe7de5e395d1d69361d229eb8d5ccc..71fc3322136f2f1c4e73225922b86c926befb9d7 100644 --- a/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py +++ b/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py @@ -12,7 +12,7 @@ import logging from PyConf import configurable from PyConf.application import default_raw_event, default_raw_banks from PyConf.utilities import DISABLE_TOOL -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.Algorithms import ( fromPrUpstreamTracksV1Tracks, diff --git a/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py b/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py index 54ec24f8b67ee2e431df321421f72d12776e467f..3ccd7fbb0ccb439d5e0ac8a1bb5c8c1d79629f3b 100644 --- a/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py +++ b/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py @@ -25,7 +25,7 @@ from .protoparticles import ( ) from PyConf import configurable -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.Algorithms import (TrackContainerCopy, RecSummaryMaker, TrackContainersMerger, TracksEmptyProducer, PVsEmptyProducer, RecVertexEmptyProducer, diff --git a/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py b/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py index 945b7faafda099308e11703fff633b07de7fc858..8e5866a4dc9c7219a6721c91214ed80221c791c6 100644 --- a/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py +++ b/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from PyConf import configurable -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.Algorithms import (LHCb__Converters__Track__SOA__fromV1Track as TrackSOAFromV1, fromV2MuonPIDV1MuonPID, diff --git a/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py b/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py index 5b2f467b53b845278ab3ce0f3dc801b57a37f809..30918eb186934b7846a901bd94fdb4461b59c02c 100644 --- a/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py +++ b/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py @@ -9,25 +9,11 @@ # or submit itself to any jurisdiction. # ############################################################################### from GaudiConf import reading -from GaudiConf.PersistRecoConf import persisted_location - -_reco_loc = { - "ChargedProtos": ("/Event/HLT2/Rec/ProtoP/Charged", "ProtoParticles"), - "NeutralProtos": ("/Event/HLT2/Rec/ProtoP/Neutrals", "ProtoParticles"), - "Tracks": ("/Event/HLT2/Rec/Track/Best", "Tracks"), - "PVs": ("/Event/HLT2/Rec/Vertex/Primary", "PVs"), - "CaloElectrons": ("/Event/HLT2/Rec/Calo/Electrons", "CaloHypos"), - "CaloPhotons": ("/Event/HLT2/Rec/Calo/Photons", "CaloHypos"), - "CaloMergedPi0s": ("/Event/HLT2/Rec/Calo/MergedPi0s", "CaloHypos"), - "CaloSplitPhotons": ("/Event/HLT2/Rec/Calo/SplitPhotons", "CaloHypos"), - "MuonPIDs": ("/Event/HLT2/Rec/Muon/MuonPID", "MuonPIDs"), - "MuonPIDTracks": ("/Event/HLT2/Rec/Muon/MuonTracks", "Tracks"), - "RichPIDs": ("/Event/HLT2/Rec/Rich/PIDs", "RichPIDs"), - "RecSummary": ("/Event/HLT2/Rec/Summary", "RecSummary") -} - - -def upfront_reconstruction(manifest): +from PyConf.packing import persisted_location, reco_locations, pp2mcp_locations +from PyConf.reading import upfront_decoder + + +def upfront_reconstruction(simulation=True): """Return a list DataHandles that define the upfront reconstruction output. This differs from `reconstruction` as it should not be used as inputs to @@ -37,41 +23,44 @@ def upfront_reconstruction(manifest): """ stream = '/Event/HLT2' - decoder = reading.decoder( - configurables=False, output_level=4, stream=stream) - - mc_algs = reading.mc_unpackers( - process='Hlt2', configurables=False, output_level=4) inv_map = {v: k for k, v in reading.type_map().items()} - # make sure that the reco locations are spliced into the provided manifest... - dl = {v[0]: v[1] for v in manifest['PackedLocations']} - dl.update((v[0], inv_map[v[1]]) for v in _reco_loc.values()) + reco_loc = reco_locations(stream) + dl = {v[0]: inv_map[v[1]] for v in reco_loc.values()} + + mc_algs = [] + if simulation: + mc_algs = reading.mc_unpackers(process='Hlt2', configurables=False) + + pp2mcp_loc = pp2mcp_locations(stream) + dl.update((v[0], inv_map[v[1]]) for v in pp2mcp_loc.values()) + + # As reco locations have predefined types, there is no need for a manifest file + # To use same functionality as in reding.py, make a manifest from known locations/types m = {'PackedLocations': [(k, v) for k, v in dl.items()]} ## TODO: only pass manifest _once_ into reading.whatever -- i.e. `unpackers` can call make_locations itself... unpackers = reading.unpackers( reading.make_locations(m, stream), m, - decoder.OutputBuffers, - configurables=False, - mc=mc_algs, - output_level=4) + upfront_decoder("Hlt2"), + configurables=False) ### TODO:FIXME take advantage of the fact that the above have datahandles... # i.e. should _not_ have to return decoder here, and should just return the _output handles_ and not the algorithms # i.e. `upfront_reconstruction` should be a drop-in replacement for `reconstruction()`, with the same return type - return [decoder] + mc_algs + unpackers + return [upfront_decoder("Hlt2").producer] + mc_algs + unpackers -def reconstruction(manifest): +def reconstruction(simulation=True): """Return a {name: DataHandle} dict that define the reconstruction output.""" data = {} - unpackers = upfront_reconstruction(manifest) + unpackers = upfront_reconstruction(simulation) + reco_loc = reco_locations("/Event/HLT2") - for key, value in _reco_loc.items(): + for key, value in reco_loc.items(): for v in unpackers: if "OutputName" in v.outputs.keys( ) and v.OutputName.location == value[0]: @@ -90,19 +79,3 @@ def reconstruction(manifest): "Output": persisted_location('RecSummary') }).Output return data - - -def make_charged_protoparticles(): - return reconstruction()['ChargedProtos'] - - -def make_neutral_protoparticles(): - return reconstruction()['NeutralProtos'] - - -def make_pvs(): - return reconstruction()['PVs'] - - -def make_tracks(): - return reconstruction()['Tracks'] diff --git a/Hlt/RecoConf/python/RecoConf/reco_objects_from_file.py b/Hlt/RecoConf/python/RecoConf/reco_objects_from_file.py index 254797dc83615abd8c95bb19630493455f608789..252a4f8675d9e709d3fa1f9b55a493d89b079680 100644 --- a/Hlt/RecoConf/python/RecoConf/reco_objects_from_file.py +++ b/Hlt/RecoConf/python/RecoConf/reco_objects_from_file.py @@ -10,7 +10,7 @@ ############################################################################### from .data_from_file import reco_unpackers from PyConf.Algorithms import FakeRecSummaryMaker -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location def upfront_reconstruction(): diff --git a/Hlt/RecoConf/python/RecoConf/reconstruction_objects.py b/Hlt/RecoConf/python/RecoConf/reconstruction_objects.py index 6a38fed91531acb07bb370dcbb7b301fa11ccc25..77f2632d20624ddd1733dda0205bcd0323843476 100644 --- a/Hlt/RecoConf/python/RecoConf/reconstruction_objects.py +++ b/Hlt/RecoConf/python/RecoConf/reconstruction_objects.py @@ -26,30 +26,21 @@ from RecoConf.reco_objects_for_spruce import ( ) from RecoConf.hlt2_global_reco import reconstruction as reconstruction_from_reco -from GaudiConf.reading import load_manifest @configurable(cached=True) -def reconstruction(from_file=True, - spruce=False, - manifest=None, - manifest_file=None): +def reconstruction(from_file=True, spruce=False): """Return reconstruction objects. Note it is advised to use this function if more than one object is needed, rather than the accessors below as it makes the configuration slower. """ - if manifest_file: - assert not manifest, "must only specify one of manifest and manifest_file" - manifest = load_manifest(manifest_file) if spruce: assert from_file, 'For sprucing, from_file must be set to True (default value)' if from_file: if spruce: - assert manifest, 'For sprucing, a manifest with file contents must be specified' - reco = reconstruction_for_spruce(manifest) - upfront_reconstruction = upfront_reconstruction_for_spruce( - manifest) + reco = reconstruction_for_spruce() + upfront_reconstruction = upfront_reconstruction_for_spruce() else: reco = reconstruction_from_file() upfront_reconstruction = upfront_reconstruction_from_file() @@ -103,3 +94,45 @@ def make_tracks(): """ return reconstruction()["Tracks"] + + +def make_rich_pids(): + """Return a DataHandle to the container of RichPIDS + + """ + return reconstruction()['RichPIDs'] + + +def make_muon_pids(): + """Return a DataHandle to the container of MuonPIDs + + """ + return reconstruction()['MuonPIDs'] + + +def make_calo_electrons(): + """Return a DataHandle to the container of Calo Electrons + + """ + return reconstruction()['CaloElectrons'] + + +def make_calo_photons(): + """Return a DataHandle to the container of Calo Photons + + """ + return reconstruction()['CaloPhotons'] + + +def make_calo_splitphotons(): + """Return a DataHandle to the container of Calo SplitPhotons + + """ + return reconstruction()['CaloSplitPhotons'] + + +def make_calo_mergedPi0s(): + """Return a DataHandle to the container of Calo Merged Pi0s + + """ + return reconstruction()['CaloMergedPi0ss'] diff --git a/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py b/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py index fa6a40293f3400744d67c9c20847030baa42af1c..cc659560e43f437b66c233b4debe59c260d1b624 100644 --- a/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py +++ b/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from PyConf import configurable -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.application import default_raw_event from Configurables import Rich__Future__ParticleProperties as PartProps diff --git a/Hlt/RecoConf/python/RecoConf/standalone.py b/Hlt/RecoConf/python/RecoConf/standalone.py index efc299062f533274dfe60bdbe92e0885830b9b24..1173054a1b0e9bb692b7cf3e8a1a65dc8ec0b27b 100644 --- a/Hlt/RecoConf/python/RecoConf/standalone.py +++ b/Hlt/RecoConf/python/RecoConf/standalone.py @@ -41,7 +41,7 @@ from .calorimeter_mc_checking import ( from .rich_reconstruction import make_rich_pixels, default_rich_reco_options from PyConf.application import default_raw_event, make_odin, make_data_with_FetchDataFromFile -from GaudiConf.PersistRecoConf import persisted_location +from PyConf.packing import persisted_location from PyConf.Algorithms import ( VeloRetinaClusterTrackingSIMD, CaloFutureDigit2MCLinks2Table, diff --git a/doc/tutorials/hlt2_analysis.rst b/doc/tutorials/hlt2_analysis.rst index 77e6eca95d7a6688e0ab11dd1481f01903472ff6..5002ef29e393490af11d898f7dc1f84b446f9235 100644 --- a/doc/tutorials/hlt2_analysis.rst +++ b/doc/tutorials/hlt2_analysis.rst @@ -131,8 +131,7 @@ familiar to you:: decoder = reading.decoder() - unpackers = reading.unpackers( - locations, manifest, decoder.OutputBuffers, mc=mc_unpackers) + unpackers = reading.unpackers(locations, manifest, decoder.OutputBuffers) reading_algs += [decoder] reading_algs += mc_unpackers