diff --git a/Hlt/Hlt2Conf/CMakeLists.txt b/Hlt/Hlt2Conf/CMakeLists.txt index f93c9c9435838cf8bef65f28fad0756eaab68694..a60aea47f990812d7043d9f20d2a4dc3daf62fa4 100644 --- a/Hlt/Hlt2Conf/CMakeLists.txt +++ b/Hlt/Hlt2Conf/CMakeLists.txt @@ -110,23 +110,10 @@ if(BUILD_TESTING AND NOT USE_DD4HEP) Hlt2Conf.sprucing.test_turcal_spruce_2022_data Hlt2Conf.sprucing.test_turcal_spruce_2022_data_pid_check Hlt2Conf.sprucing.test_turcal_spruce_2022_data_pid_rb_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_trackeff_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_trackeff_rb_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_monitoring_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_monitoring_rb_check Hlt2Conf.sprucing.test_excl_spruce_2022_data_b2oc_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_bandq_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_qee_check Hlt2Conf.sprucing.test_excl_spruce_2022_data_rd_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_sl_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_b2cc_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_b2oc_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_bandq_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_bnoc_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_charm_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_qee_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_rd_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_sl_check Hlt2Conf.sprucing.test_excl_spruce_2023_1_data Hlt2Conf.sprucing.test_excl_spruce_2023_2_data Hlt2Conf.sprucing.test_hlt2_foroverlapcheck diff --git a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml index 6f3676087245658d6e3210be623d8f59ade7b431..db386a2b248885466946aa5d6bf5f280d10382fe 100644 --- a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml +++ b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml @@ -10,8 +10,7 @@ evt_max: -1 geometry_version: run3/trunk conditions_version: jonrob/all-pmts-active - -process: Spruce +process: TurboSpruce input_raw_format : 0.5 dddb_tag: dddb-20231017 diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 6aa2ec412417816663f827bde15f43d0f73aefc5..f7076a394fe9894f420dcdb019a6b2177f43b17a 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -284,7 +284,6 @@ def turbo_overlap(options: Options): This test shows the overlap using PassLine on a data file in dpa eos space """ - create_or_reuse_rootIOAlg(options) def make_streams(): stream_A = Stream( @@ -304,6 +303,8 @@ def turbo_overlap(options: Options): def spruce_overlap(options: Options): + import json + """ Test for the technical overlap of passthrough streams created in the sprucing-streaming step @@ -315,24 +316,38 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ - create_or_reuse_rootIOAlg(options) - - input_line_config = options.input_streams_attributes_file + from pathlib import Path - spruce_streams = { - "streamone": ["Hlt2.*one.*"], - "streamtwo": ["Hlt2.*two.*"], - "streamthree": ["Hlt2.*three.*"], - } + turbolinedict = Path(__file__).parent / "test_streaming_config.json" + stream = "default" + input_line_config = options.input_streams_attributes_file + stream_config = turbolinedict + + # Open and merge the line_config and the streaming_config + with open(stream_config, "r") as json_file: + streaming_config = json.load(json_file) + with open(input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] + + for line, l_config in line_config.items(): + for stream, s_config in streaming_config.items(): + if line in s_config: + l_config["stream"] = stream + break + else: + raise Exception(f"Line {line} not found in any stream") + + assert sum(map(len, streaming_config.values())) == len(line_config.keys()), ( + f"There are {len(line_config.keys())} lines to be run but only {sum(map(len, streaming_config.values()))} lines in the streams config" + ) + + print(f"line_config: {line_config}") custom_prescales = {"Hlt2Linetwo": 0.5} def make_streams(): return turbo_spruce( - input_line_config, - spruce_streams, - custom_prescales=custom_prescales, - stream="default", + line_config, custom_prescales=custom_prescales, use_regex=False ) with list_of_full_stream_lines.bind(lines=[]): diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py index 6e15452dac952ea57dbbf0073973fc53a5e7601b..4399f24bc761877586edaed8e2aa0a306665a3a2 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py @@ -157,17 +157,20 @@ turbolinedict = { from Moore.production import turbo_spruce -def make_turbo_spruce_prod_streams(input_streams_attributes_file): - # create_or_reuse_rootIOAlg(options) +def make_turbo_spruce_prod_streams(line_config): + return turbo_spruce(line_config, use_regex=True, spruce_streams=turbolinedict) - # custom_prescales = {"Hlt2Linetwo": 0.5} - return turbo_spruce(input_streams_attributes_file, turbolinedict) +def turbo_spruce_production(options: Options): + import json + input_line_config = options.input_streams_attributes_file + stream = "turbo" + with open(input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] -def turbo_spruce_production(options: Options): def make_streams(): - return make_turbo_spruce_prod_streams(options.input_streams_attributes_file) + return make_turbo_spruce_prod_streams(line_config) config = run_moore(options, make_streams, public_tools=[]) return config diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8a61b1beed0e8c2be2b2dcee199a7bffb2e05b16 --- /dev/null +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json @@ -0,0 +1 @@ +{"streamone": ["Hlt2Lineone_extraoutputs"], "streamtwo": ["Hlt2Linetwo"], "streamthree": ["Hlt2Linethree"]} diff --git a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py index 0f7edb7d21c94ef1b92fb0b3327bb273bdeef643..13a0a863429502930f9bffdf38d48864f2b168b6 100644 --- a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py +++ b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py @@ -27,6 +27,7 @@ from RecoConf.reconstruction_objects import reconstruction log = logging.getLogger() options.input_process = "Hlt2" +options.process = "TurboSpruce" fname_helper = FileNameHelper(process="spruce", stream_config="turbo") fname_helper.make_tmp_dirs() @@ -38,8 +39,14 @@ options.input_streams_attributes_file = FileNameHelper( ).streams_attributes_file() +input_line_config = options.input_streams_attributes_file +stream = "turbo" +with open(input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] + + def make_streams(): - real_streams = make_turbo_spruce_prod_streams(options.input_streams_attributes_file) + real_streams = make_turbo_spruce_prod_streams(line_config) # Write out stream configuration to JSON file for use later in the test with open(fname_helper.stream_config_json_path(), "w") as f: diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py index e5c45f44b5bfd4058217b37bb6554ef3ee9ab440..dbe1e1c90c204384df39c9a443da0966ff1b341d 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py @@ -49,7 +49,7 @@ assert args.p == "Spruce" or args.p == "Turbo", ( if args.p == "Turbo": RECO_ROOT = "/Event/HLT2" else: - RECO_ROOT = "/Event/Spruce/HLT2" + RECO_ROOT = "/Event/Turbo/HLT2" cf_node = do_unpacking(input_process=args.p, has_mc_data=options.simulation) @@ -119,12 +119,12 @@ for ii in range(nevents): else: # Spruce if options.input_stream == "streamone": # Check particles and rb - check_particles(TES, f"/Event/Spruce/SprucePass_{lineone}") + check_particles(TES, f"/Event/Turbo/{lineone}") check_banks(TES, options.input_stream, [9]) # Check persistreco for reco in reco_locs: - persistreco = TES[f"/Event/Spruce/HLT2/Rec/{reco}"] + persistreco = TES[f"/Event/Turbo/HLT2/Rec/{reco}"] if not persistreco or persistreco.size() <= 0: raise RuntimeError( "Check ERROR - Reco locations not propagated" @@ -139,9 +139,7 @@ for ii in range(nevents): ## proto().track().ancestors().size() # Check CALO clusters - caloclusters = TES["/Event/Spruce/HLT2/Rec/Calo/Electrons"][ - 0 - ].clusters() + caloclusters = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][0].clusters() if not caloclusters or caloclusters.size() <= 0: raise RuntimeError("Check ERROR - Calo clusters not propagated") else: @@ -150,7 +148,7 @@ for ii in range(nevents): ) # Check CALO digits - calodigits = TES["/Event/Spruce/HLT2/Rec/Calo/Electrons"][0].digits() + calodigits = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][0].digits() if not calodigits or calodigits.size() <= 0: raise RuntimeError("Check ERROR - Calo digits not propagated") else: @@ -159,7 +157,7 @@ for ii in range(nevents): ) # Check PV tracks - pvtracks = TES["/Event/Spruce/HLT2/Rec/Vertex/Primary"][0].tracks() + pvtracks = TES["/Event/Turbo/HLT2/Rec/Vertex/Primary"][0].tracks() if not pvtracks or pvtracks.size() <= 0: raise RuntimeError("Check ERROR - PV tracks not propagated") else: @@ -169,30 +167,28 @@ for ii in range(nevents): # Check extra_outputs for extraoutput in lineone_extraoutputs: - check_particles( - TES, f"/Event/Spruce/SprucePass_{lineone}/{extraoutput}" - ) + check_particles(TES, f"/Event/Turbo/{lineone}/{extraoutput}") # Check no overlap try: - check_particles(TES, f"/Event/Spruce/SprucePass_{linetwo}") + check_particles(TES, f"/Event/Turbo/{linetwo}") except RuntimeError: print(f"Check - No particles from {linetwo} AS EXPECTED") else: print("Check ERROR - Overlap is present") finally: - print(f"Done with SprucePass_{lineone}") + print(f"Done with {lineone}") break else: # streamtwo # Check particles and no Rich rb - check_particles(TES, f"/Event/Spruce/SprucePass_{linetwo}") + check_particles(TES, f"/Event/Turbo/{linetwo}") check_not_banks(TES, options.input_stream, [9]) # Check persistreco for reco in reco_locs: - persistreco = TES[f"/Event/Spruce/HLT2/Rec/{reco}"] + persistreco = TES[f"/Event/Turbo/HLT2/Rec/{reco}"] if not persistreco or persistreco.size() <= 0: print(f"Reco ({reco}) not propagated as expected") else: @@ -203,7 +199,7 @@ for ii in range(nevents): # Cannot check absense of CALO objs with linetwo # Check PV tracks - pvtracks = TES["/Event/Spruce/HLT2/Rec/Vertex/Primary"][0].tracks() + pvtracks = TES["/Event/Turbo/HLT2/Rec/Vertex/Primary"][0].tracks() if not pvtracks or pvtracks.size() == 0: print("PV tracks not propagated as expected") else: @@ -213,11 +209,11 @@ for ii in range(nevents): # Check for no overlap try: - check_particles(TES, f"/Event/Spruce/SprucePass_{lineone}") + check_particles(TES, f"/Event/Turbo/{lineone}") except RuntimeError: print(f"Check - No particles from {lineone} AS EXPECTED") else: print("Check ERROR - Overlap is present") finally: - print(f"Done with SprucePass_{linetwo}") + print(f"Done with {linetwo}") break diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt deleted file mode 100644 index 4da1e673aacbbe0197a8f84e8ef0d01f87e795dd..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.bandq.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - bandq - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt deleted file mode 100644 index 1a34f56916949e098e14e65b099a564c7b1c1a32..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.qee.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - qee - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt deleted file mode 100644 index e2b670a900963d0796d62a8f8d0b947fe9a7fb85..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.sl.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - sl - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt deleted file mode 100644 index 68da553c431fb8f6ceca85a82a628d616a33ec79..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.b2cc.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - b2cc - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt deleted file mode 100644 index 24c9e0716ee5896053c015bee2b4797ef56277dd..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.bandq.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - bandq - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt deleted file mode 100644 index 68ae8fd7fa2dc46df0b3c147dcd69e3e36bb393e..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.bnoc.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - bnoc - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt deleted file mode 100644 index f9343c58cb321316751ef205a195794073012c04..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.charm.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - charm - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt deleted file mode 100644 index 08efc243515a6782765bf7793dc20782926ef401..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.qee.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - qee - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt deleted file mode 100644 index 8578829b0de95e09c83c2cabc8ea9544e4c9b784..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.sl.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - sl - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt index 059a1c7a04631afb9ce74795d74187bcccef1250..3cf1238134d357ba8093bd1620c911531d30c926 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt @@ -34,10 +34,10 @@ import re ref_file=str(open("test_hlt2_foroverlapcheck.stdout", "r").read()) ref_matches_lineone = re.findall('LAZY_AND: (Hlt2Lineone_extraoutputs) .*Sum=(\d+)', ref_file) -matches_lineone = re.findall('(VoidFilter/SprucePass_Hlt2Lineone_extraoutputs_Hlt2Filter) .*Sum=(\d+)', stdout) +matches_lineone = re.findall('(VoidFilter/Hlt2Lineone_extraoutputs_Hlt2Filter) .*Sum=(\d+)', stdout) ref_matches_linetwo = re.findall('LAZY_AND: (Hlt2Linetwo) .*Sum=(\d+)', ref_file) -matches_linetwo = re.findall('(VoidFilter/SprucePass_Hlt2Linetwo_Hlt2Filter) .*Sum=(\d+)', stdout) +matches_linetwo = re.findall('(VoidFilter/Hlt2Linetwo_Hlt2Filter) .*Sum=(\d+)', stdout) print(f"ref_matches_lineone is {ref_matches_lineone} and matches_lineone is {matches_lineone}") print(f"ref_matches_linetwo is {ref_matches_linetwo} and matches_linetwo is {matches_linetwo}") @@ -51,7 +51,7 @@ else: print(f"HLT2 found {ref_matches_lineone[0][1]} events and passthrough Sprucing found {matches_lineone[0][1]} for line {ref_matches_lineone[0][0]} - all good") -prescale_linetwo = re.findall('(DeterministicPrescaler/SprucePass_Hlt2Linetwo_Prescaler) .*Eff=..\s(\d+.\d+)', stdout) +prescale_linetwo = re.findall('(DeterministicPrescaler/Hlt2Linetwo_Prescaler) .*Eff=..\s(\d+.\d+)', stdout) print(f"Line two prescale is {prescale_linetwo}") if not (float(prescale_linetwo[0][1]) > 40. and float(prescale_linetwo[0][1]) < 60.): diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt deleted file mode 100644 index 16ff0c69be2ea406c2ce11774921fe26fb69a0ac..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.monitoring.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - monitoring - -rb_to_check - 16 - -rb_to_check_not - 73649138477 - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt deleted file mode 100644 index 1fd0294f911010aed2e4a001e16e9ee53251a14e..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.monitoring_raw.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - monitoring_raw - -rb_to_check - 1673649138477 - -true -../refs/test_turcal_spruce_2022_data_monitoring_rb_check.ref - - -from Moore.qmtest.exclusions import remove_known_warnings -from GaudiConf.QMTest.LHCbExclusions import preprocessor -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 1}, - stdout=remove_known_warnings(stdout)) -validateWithReference(preproc=preprocessor) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt deleted file mode 100644 index 1c3e89f9a18bbede57a5ba44408e147e1a89538d..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.trackeff.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - trackeff - -rb_to_check - 16 - -rb_to_check_not - 73649138477 - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt deleted file mode 100644 index d413f48ade6cbc71eac1ad707c1a4eb55f45fe20..0000000000000000000000000000000000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.trackeff_raw.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - trackeff_raw - -rb_to_check - 1673649138477 - -true -../refs/test_turcal_spruce_2022_data_trackeff_rb_check.ref - - -from Moore.qmtest.exclusions import remove_known_warnings -from GaudiConf.QMTest.LHCbExclusions import preprocessor -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 1}, - stdout=remove_known_warnings(stdout)) -validateWithReference(preproc=preprocessor) - - - diff --git a/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref b/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref index 724e6d82bf4436e58795f9a3395395ba38024a09..afdca68234449fa18cfa86f5effb2105800c20d8 100644 --- a/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref +++ b/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref @@ -49,6 +49,7 @@ rb_to_check_not None |-phoenix_filename = '' (default: '') |-preamble_algs = [] (default: []) |-print_freq = 10000 (default: 10000) +|-process = None |-python_logging_level = 20 (default: 20) |-require_specific_decoding_keys = [] (default: []) |-root_ioalg_name = 'RootIOAlg' (default: 'RootIOAlg') diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 839a2409decdefe2f0b55e00c2a0560f5e7ece79..0f252926bfd47b250ab796d4fa8e2adb95c07e8b 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -26,6 +26,7 @@ class ProcessTypes(str, Enum): Hlt1 = "Hlt1" Hlt2 = "Hlt2" TurboPass = "TurboPass" + TurboSpruce = "TurboSpruce" Spruce = "Spruce" ReconstructionOnly = "ReconstructionOnly" Monitoring = "Monitoring" @@ -34,7 +35,7 @@ class ProcessTypes(str, Enum): class Options(DefaultOptions): tck: Optional[int] = None - process: ProcessTypes = ProcessTypes.Hlt2 + process: Optional[ProcessTypes] = None @contextmanager def apply_binds(self): @@ -64,7 +65,11 @@ class Options(DefaultOptions): persistreco_version.global_bind(version=self.persistreco_version) - if self.process == ProcessTypes.Spruce: + if ( + self.process == ProcessTypes.Spruce + or self.process == ProcessTypes.TurboSpruce + or self.process == ProcessTypes.TurboPass + ): reconstruction.global_bind(spruce=True, from_file=True) with super().apply_binds(): diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 08409e21a768f2070dfe27639255e83d0729d597..f79b3325b7e70e1039c35a8d11f9d08d9e2bf9e1 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -135,10 +135,18 @@ def moore_control_flow(options, streams, process, analytics=False): extra_locations_to_persist = [] packed_data = {} line_output_cf = {} + if process in ["hlt2", "spruce"]: - event_output_prefix = "/Event/Spruce" if process == "spruce" else "/Event/HLT2" - reco_output_prefix = "/Event/HLT2" + if process == "hlt2": + event_output_prefix = reco_output_prefix = "/Event/HLT2" + + if process == "spruce": + event_output_prefix = "/Event/Spruce" + if getattr(options, "process", None) and options.process == "TurboSpruce": + event_output_prefix = "/Event/Turbo" + reco_output_prefix = "/Event/HLT2" + ## The next bit is indented (line_output_cf, extra_locations_to_persist, packed_data) = ( persist_line_outputs( streams=streams, @@ -292,18 +300,22 @@ def run_moore( options.output_streams_attributes_file, streams ) - assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( - "Expected exclusively all Hlt1, all Hlt2, all Spruce or all Pass lines" - ) - - if hlt1: - process = "hlt1" - elif hlt2: - process = "hlt2" - elif spruce: + if getattr(options, "process", None) and options.process == "TurboSpruce": process = "spruce" - elif passthrough: - process = "pass" + + else: + assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( + "Expected exclusively all Hlt1, all Hlt2, all Spruce or all Pass lines" + ) + + if hlt1: + process = "hlt1" + elif hlt2: + process = "hlt2" + elif spruce: + process = "spruce" + elif passthrough: + process = "pass" # Combine all lines and output in a global control flow. moore_control_node = moore_control_flow(options, streams, process, analytics) diff --git a/Hlt/Moore/python/Moore/lines.py b/Hlt/Moore/python/Moore/lines.py index 762387f20d94cf4c2e0a2dca133e54fc4ae813c4..a3338ac2e241c9afb077b3f4772d83743e777a0a 100644 --- a/Hlt/Moore/python/Moore/lines.py +++ b/Hlt/Moore/python/Moore/lines.py @@ -378,6 +378,7 @@ class Hlt2Line(DecisionLine): extra_outputs (iterable of 2-tuple): List of `(name, DataHandle)` pairs. persistreco (bool): If True, request HLT2 reconstruction persistence. hlt1_filter_code (list(str)): string used to define a HLT1 filter. + stream (str): name of the stream to which the line belongs, defaults to None. Attributes: objects_to_persist (list of DataHandle): Objects which this lines @@ -412,6 +413,7 @@ class Hlt2Line(DecisionLine): algs, prescale=1.0, postscale=1.0, + stream=None, extra_outputs=None, persistreco=False, tagging_particles=False, @@ -461,12 +463,14 @@ class Hlt2Line(DecisionLine): else: algs += monitoring_algs super(Hlt2Line, self).__init__(name, algs, prescale, postscale) - if not self.name.startswith(self._CLASS_NAME_PREFIX): - raise ValueError( - "name {!r} does not start with {!r}".format( - name, self._CLASS_NAME_PREFIX + if not isinstance(self, SpruceLine): + if not self.name.startswith(self._CLASS_NAME_PREFIX): + raise ValueError( + "name {!r} does not start with {!r}".format( + name, self._CLASS_NAME_PREFIX + ) ) - ) + self.extra_outputs = tuple(sorted(set(extra_outputs or []), key=hash)) self.raw_banks = tuple(sorted(set(raw_banks or []), key=hash)) self.persistreco = persistreco @@ -475,6 +479,7 @@ class Hlt2Line(DecisionLine): self.calo_clusters = calo_clusters self.pv_tracks = True if persistreco else pv_tracks self.track_ancestors = track_ancestors + self.stream = stream # The line guarantees that these objects will be present in the TES if # this line made a positive decision self.objects_to_persist = [] @@ -507,6 +512,7 @@ class Hlt2Line(DecisionLine): "track_ancestors": self.track_ancestors, "raw_banks": self.raw_banks, "produces_particles": is_output_producer, + "stream": self.stream, } def to_dict(self): @@ -732,6 +738,7 @@ class SpruceLine(Hlt2Line): algs, prescale=1.0, postscale=1.0, + stream=None, extra_outputs=None, raw_banks=None, persistreco=False, @@ -747,10 +754,7 @@ class SpruceLine(Hlt2Line): self.hlt2_filter_code = hlt2_filter_code if hlt2_filter_code: hlt2_filter = _return_filter( - f"{name}_Hlt2Filter", - self.hlt2_filter_code, - self._HLT2_FILTER_SOURCE_ID, - False if isinstance(self, PassLine) else True, + f"{name}_Hlt2Filter", self.hlt2_filter_code, self._HLT2_FILTER_SOURCE_ID ) algs = [hlt2_filter] + algs @@ -761,6 +765,7 @@ class SpruceLine(Hlt2Line): algs=algs, prescale=prescale, postscale=postscale, + stream=stream, extra_outputs=extra_outputs, raw_banks=raw_banks, persistreco=persistreco, @@ -772,12 +777,6 @@ class SpruceLine(Hlt2Line): hlt1_filter_code=hlt1_filter_code, monitoring_variables=monitoring_variables, ) - if not self.name.startswith(self._CLASS_NAME_PREFIX): - raise ValueError( - "name {!r} does not start with {!r}".format( - name, self._CLASS_NAME_PREFIX - ) - ) class PassLine(SpruceLine): diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index 8a0897d22b0d166fe5ad64dffb301ca02a437945..e49c28f43f1eb51a7deafd37986e0f8f036cda2a 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -519,58 +519,74 @@ def _spruce( return config -def turbo_spruce( - input_line_config, spruce_streams, custom_prescales={}, stream="turbo" -): +def turbo_spruce(line_config, custom_prescales={}, use_regex=True, spruce_streams=None): """ - Creates and configures sprucing streams for the passthrough of HLT2 TURBO output. + Creates and configures sprucing streams for HLT2 TURBO output. - This function reads the attributes of all Hlt2Lines - from a JSON file made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their attributes and - the given regex patterns in `spruce_streams`. If required it can apply custom prescales. + This function reads the attributes of Hlt2Lines from a dict made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their "stream" attribute OR by regex. If required it can apply custom prescales. Args: - input_line_config (str): JSON file made at the HLT2 stage with stream and line attributes - spruce_streams (dict): A dictionary mapping stream names to regex - pattern lists for Sprucing streams. + line_config (dict): dict made at the HLT2 stage with line attributes. This dict must have the following structure: {"line_name": {"name": "line_name", "stream": "stream_name", "prescale": 1.0} ...}. See Sprucing_tests:spruce_overlap for an example custom_prescales (dict, optional): A dictionary mapping line names to prescale factors. Defaults to empty dictionary. - stream (str, optional): The stream to be configured. Defaults to "turbo". + use_regex (bool, optional): If True, stream lines according to spruce_streams regex. Defaults to True. + spruce_streams (dict, optional): A dictionary mapping stream names to regex patterns. Defaults to None. Returns: Streams: An object containing the configured sprucing streams for passthrough of TURBO output. """ + turbo_lines = list(line_config.keys()) + # print(f"Lines in HLT2 stream requested: {turbo_lines}") - with open(input_line_config, "r") as json_file: - line_attributes = json.load(json_file)[stream] + if use_regex: + assert all( + isinstance(regex_list, list) for regex_list in spruce_streams.values() + ), ( + "The `spruce_streams` dict for use_regex=True must map stream names to regex pattern LISTS" + ) + # Can stream lines according to regex + streaming = { + stream_name: [ + line + for line in turbo_lines + if any(re.match(regex, line) for regex in regex_list) + ] + for stream_name, regex_list in spruce_streams.items() + } - turbo_lines = list(line_attributes.keys()) - # print(f"Lines in HLT2 stream requested: {turbo_lines}") + else: # Using stream attribute in line_config to stream lines + list_of_streams = set( + line_config[line]["stream"] for line in line_config.keys() + ) - assert all( - isinstance(regex_list, list) for regex_list in spruce_streams.values() - ), "The `spruce_streams` dict must map stream names to regex pattern LISTS" + ### Check that all lines have a stream + if None in list_of_streams: + raise ValueError("At least one line has no stream attribute assigned!!!") + print(f"Streams found in line attributes file are : {list_of_streams}") + + # Streaming is done via exact "regex" list of lines + streaming = { + stream_name: [ + line + for line in turbo_lines + if line_config[line].get("stream") == stream_name + ] + for stream_name in list_of_streams + } - # Can stream lines according to regex - streaming = { - stream_name: [ - line - for line in turbo_lines - if any(re.match(regex, line) for regex in regex_list) - ] - for stream_name, regex_list in spruce_streams.items() - } + for stream_name in streaming.keys(): + print(f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") - # for stream_name in streaming.keys(): - # print( - # f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") + assert sum(map(len, streaming.values())) == len(line_config.keys()), ( + f"There are {len(line_config.keys())} lines to be streamed but only {sum(map(len, streaming.values()))} lines in the streams config" + ) streams = [ Stream( stream_name, lines=[ _make_pass_spruceline( - line_attributes, line, custom_prescales=custom_prescales + line_config[line], custom_prescales=custom_prescales ) for line in streaming[stream_name] ], @@ -584,13 +600,12 @@ def turbo_spruce( return my_streams -def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): +def _make_pass_spruceline(line_attributes, custom_prescales={}): """ Create a SpruceLine for a given Hlt2Line. Args: - line_attributes (dict): Attributes for all lines eg. `json[stream]` where json is read from `options.input_streams_attributes_file`. - hlt2_linename (str): The name of the Hlt2Line to be converted into a SpruceLine. + line_attributes (dict): Attributes for line. custom_prescales (dict, optional): A dictionary mapping lines to custom prescales. Defaults to empty dictionary. Returns: @@ -598,27 +613,25 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): """ from PyConf.reading import get_particles, upfront_decoder - filter = f"{hlt2_linename}Decision" - location = f"/Event/HLT2/{hlt2_linename}/Particles" - spruce_linename = hlt2_linename.replace("Hlt2", "SprucePass_Hlt2") + linename = line_attributes["name"] + filter = f"{linename}Decision" + location = f"/Event/HLT2/{linename}/Particles" + # print( - # f"Line {hlt2_linename} becomes {spruce_linename} with candidates at /Event/Spruce/{spruce_linename}/Particles" + # f"Line {linename} produces candidates at /Event/Turbo/{spruce_linename}/Particles" # ) - line_attributes = line_attributes[hlt2_linename] - line_attributes["prescale"] = custom_prescales.get(hlt2_linename, 1.0) - # print(line_attributes) + line_attributes["prescale"] = custom_prescales.get(linename, 1.0) with upfront_decoder.bind(source="Hlt2"): hlt2_particles = get_particles(location) algs = [] # Some lines do not output anything to `/Event/HLT2/{hlt2_linename}/Particles` if line_attributes["produces_particles"]: - # print(f"Line {hlt2_linename} produces location `/Event/HLT2/{hlt2_linename}/Particles`") algs.append(hlt2_particles) else: print( - f"Line {hlt2_linename} does not produce location `/Event/HLT2/{hlt2_linename}/Particles`, skipping this location" + f"Line {linename} did not produce location `/Event/HLT2/{linename}/Particles`, skipping this location" ) extra_outputs_tuplelist = [] @@ -627,10 +640,10 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): for loc in line_attributes["extra_outputs"]: # print(f"Adding extra_output {loc} for line {spruce_linename}") extra_outputs_tuplelist.append( - (loc, get_particles(f"/Event/HLT2/{hlt2_linename}/{loc}/Particles")) + (loc, get_particles(f"/Event/HLT2/{linename}/{loc}/Particles")) ) pass_spruceline = SpruceLine( - name=spruce_linename, + name=linename, hlt2_filter_code=filter, algs=algs, extra_outputs=extra_outputs_tuplelist, @@ -642,5 +655,6 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): pv_tracks=line_attributes["pv_tracks"], track_ancestors=line_attributes["track_ancestors"], raw_banks=line_attributes["raw_banks"], + stream=line_attributes["stream"], ) return pass_spruceline