From 3f10966ca3def355ecd4bdd2259c5a176afa8869 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Tue, 4 Feb 2025 19:10:11 +0100 Subject: [PATCH 01/19] change rawevent locs --- .../sprucing/lbexec_yamls/spruce_overlap.yaml | 2 +- .../python/Hlt2Conf/Sprucing_tests.py | 3 -- .../spruce_turbopass_overlap_check.py | 28 ++++++------ .../sprucing.qms/test_spruce_turbooverlap.qmt | 6 +-- Hlt/Moore/python/Moore/LbExec.py | 3 +- Hlt/Moore/python/Moore/config.py | 44 +++++++++++++------ Hlt/Moore/python/Moore/lines.py | 29 ++++++------ Hlt/Moore/python/Moore/production.py | 7 +-- 8 files changed, 70 insertions(+), 52 deletions(-) diff --git a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml index 6f367608724..6634a40e5d6 100644 --- a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml +++ b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml @@ -11,7 +11,7 @@ geometry_version: run3/trunk conditions_version: jonrob/all-pmts-active -process: Spruce +process: TurboSpruce input_raw_format : 0.5 dddb_tag: dddb-20231017 diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 6aa2ec41241..75402d09f33 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -284,7 +284,6 @@ def turbo_overlap(options: Options): This test shows the overlap using PassLine on a data file in dpa eos space """ - create_or_reuse_rootIOAlg(options) def make_streams(): stream_A = Stream( @@ -315,8 +314,6 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ - create_or_reuse_rootIOAlg(options) - input_line_config = options.input_streams_attributes_file spruce_streams = { diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py index e5c45f44b5b..ae222e8adbe 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py @@ -49,7 +49,7 @@ assert args.p == "Spruce" or args.p == "Turbo", ( if args.p == "Turbo": RECO_ROOT = "/Event/HLT2" else: - RECO_ROOT = "/Event/Spruce/HLT2" + RECO_ROOT = "/Event/Turbo/HLT2" cf_node = do_unpacking(input_process=args.p, has_mc_data=options.simulation) @@ -119,12 +119,12 @@ for ii in range(nevents): else: # Spruce if options.input_stream == "streamone": # Check particles and rb - check_particles(TES, f"/Event/Spruce/SprucePass_{lineone}") + check_particles(TES, f"/Event/Turbo/{lineone}") check_banks(TES, options.input_stream, [9]) # Check persistreco for reco in reco_locs: - persistreco = TES[f"/Event/Spruce/HLT2/Rec/{reco}"] + persistreco = TES[f"/Event/Turbo/HLT2/Rec/{reco}"] if not persistreco or persistreco.size() <= 0: raise RuntimeError( "Check ERROR - Reco locations not propagated" @@ -139,7 +139,7 @@ for ii in range(nevents): ## proto().track().ancestors().size() # Check CALO clusters - caloclusters = TES["/Event/Spruce/HLT2/Rec/Calo/Electrons"][ + caloclusters = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][ 0 ].clusters() if not caloclusters or caloclusters.size() <= 0: @@ -150,7 +150,7 @@ for ii in range(nevents): ) # Check CALO digits - calodigits = TES["/Event/Spruce/HLT2/Rec/Calo/Electrons"][0].digits() + calodigits = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][0].digits() if not calodigits or calodigits.size() <= 0: raise RuntimeError("Check ERROR - Calo digits not propagated") else: @@ -159,7 +159,7 @@ for ii in range(nevents): ) # Check PV tracks - pvtracks = TES["/Event/Spruce/HLT2/Rec/Vertex/Primary"][0].tracks() + pvtracks = TES["/Event/Turbo/HLT2/Rec/Vertex/Primary"][0].tracks() if not pvtracks or pvtracks.size() <= 0: raise RuntimeError("Check ERROR - PV tracks not propagated") else: @@ -170,29 +170,29 @@ for ii in range(nevents): # Check extra_outputs for extraoutput in lineone_extraoutputs: check_particles( - TES, f"/Event/Spruce/SprucePass_{lineone}/{extraoutput}" + TES, f"/Event/Turbo/{lineone}/{extraoutput}" ) # Check no overlap try: - check_particles(TES, f"/Event/Spruce/SprucePass_{linetwo}") + check_particles(TES, f"/Event/Turbo/{linetwo}") except RuntimeError: print(f"Check - No particles from {linetwo} AS EXPECTED") else: print("Check ERROR - Overlap is present") finally: - print(f"Done with SprucePass_{lineone}") + print(f"Done with {lineone}") break else: # streamtwo # Check particles and no Rich rb - check_particles(TES, f"/Event/Spruce/SprucePass_{linetwo}") + check_particles(TES, f"/Event/Turbo/{linetwo}") check_not_banks(TES, options.input_stream, [9]) # Check persistreco for reco in reco_locs: - persistreco = TES[f"/Event/Spruce/HLT2/Rec/{reco}"] + persistreco = TES[f"/Event/Turbo/HLT2/Rec/{reco}"] if not persistreco or persistreco.size() <= 0: print(f"Reco ({reco}) not propagated as expected") else: @@ -203,7 +203,7 @@ for ii in range(nevents): # Cannot check absense of CALO objs with linetwo # Check PV tracks - pvtracks = TES["/Event/Spruce/HLT2/Rec/Vertex/Primary"][0].tracks() + pvtracks = TES["/Event/Turbo/HLT2/Rec/Vertex/Primary"][0].tracks() if not pvtracks or pvtracks.size() == 0: print("PV tracks not propagated as expected") else: @@ -213,11 +213,11 @@ for ii in range(nevents): # Check for no overlap try: - check_particles(TES, f"/Event/Spruce/SprucePass_{lineone}") + check_particles(TES, f"/Event/Turbo/{lineone}") except RuntimeError: print(f"Check - No particles from {lineone} AS EXPECTED") else: print("Check ERROR - Overlap is present") finally: - print(f"Done with SprucePass_{linetwo}") + print(f"Done with {linetwo}") break diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt index 059a1c7a046..3cf1238134d 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_spruce_turbooverlap.qmt @@ -34,10 +34,10 @@ import re ref_file=str(open("test_hlt2_foroverlapcheck.stdout", "r").read()) ref_matches_lineone = re.findall('LAZY_AND: (Hlt2Lineone_extraoutputs) .*Sum=(\d+)', ref_file) -matches_lineone = re.findall('(VoidFilter/SprucePass_Hlt2Lineone_extraoutputs_Hlt2Filter) .*Sum=(\d+)', stdout) +matches_lineone = re.findall('(VoidFilter/Hlt2Lineone_extraoutputs_Hlt2Filter) .*Sum=(\d+)', stdout) ref_matches_linetwo = re.findall('LAZY_AND: (Hlt2Linetwo) .*Sum=(\d+)', ref_file) -matches_linetwo = re.findall('(VoidFilter/SprucePass_Hlt2Linetwo_Hlt2Filter) .*Sum=(\d+)', stdout) +matches_linetwo = re.findall('(VoidFilter/Hlt2Linetwo_Hlt2Filter) .*Sum=(\d+)', stdout) print(f"ref_matches_lineone is {ref_matches_lineone} and matches_lineone is {matches_lineone}") print(f"ref_matches_linetwo is {ref_matches_linetwo} and matches_linetwo is {matches_linetwo}") @@ -51,7 +51,7 @@ else: print(f"HLT2 found {ref_matches_lineone[0][1]} events and passthrough Sprucing found {matches_lineone[0][1]} for line {ref_matches_lineone[0][0]} - all good") -prescale_linetwo = re.findall('(DeterministicPrescaler/SprucePass_Hlt2Linetwo_Prescaler) .*Eff=..\s(\d+.\d+)', stdout) +prescale_linetwo = re.findall('(DeterministicPrescaler/Hlt2Linetwo_Prescaler) .*Eff=..\s(\d+.\d+)', stdout) print(f"Line two prescale is {prescale_linetwo}") if not (float(prescale_linetwo[0][1]) > 40. and float(prescale_linetwo[0][1]) < 60.): diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 839a2409dec..68686336724 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -27,6 +27,7 @@ class ProcessTypes(str, Enum): Hlt2 = "Hlt2" TurboPass = "TurboPass" Spruce = "Spruce" + TurboSpruce = "TurboSpruce" ReconstructionOnly = "ReconstructionOnly" Monitoring = "Monitoring" UserDefined = "UserDefined" @@ -64,7 +65,7 @@ class Options(DefaultOptions): persistreco_version.global_bind(version=self.persistreco_version) - if self.process == ProcessTypes.Spruce: + if (self.process == ProcessTypes.Spruce or self.process == ProcessTypes.TurboSpruce): reconstruction.global_bind(spruce=True, from_file=True) with super().apply_binds(): diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 08409e21a76..7c806a551cc 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -135,10 +135,20 @@ def moore_control_flow(options, streams, process, analytics=False): extra_locations_to_persist = [] packed_data = {} line_output_cf = {} - if process in ["hlt2", "spruce"]: - event_output_prefix = "/Event/Spruce" if process == "spruce" else "/Event/HLT2" - reco_output_prefix = "/Event/HLT2" + if process in ["hlt2", "spruce"]: + if process == "hlt2": + event_output_prefix = reco_output_prefix = "/Event/HLT2" + + if process =="spruce": + event_output_prefix = "/Event/Spruce" + if getattr(options, "process", None): + if options.process == "TurboSpruce": + event_output_prefix = "/Event/Turbo" + print("event_output_prefix", event_output_prefix) + reco_output_prefix = "/Event/HLT2" + + ## The next bit is indented!!!! (line_output_cf, extra_locations_to_persist, packed_data) = ( persist_line_outputs( streams=streams, @@ -292,18 +302,24 @@ def run_moore( options.output_streams_attributes_file, streams ) - assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( - "Expected exclusively all Hlt1, all Hlt2, all Spruce or all Pass lines" - ) + if getattr(options, "process", None): + if options.process == "TurboSpruce": + print("here we are") + process="spruce" + + else: + assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( + "Expected exclusively all Hlt1, all Hlt2, all Spruce or all Pass lines" + ) - if hlt1: - process = "hlt1" - elif hlt2: - process = "hlt2" - elif spruce: - process = "spruce" - elif passthrough: - process = "pass" + if hlt1: + process = "hlt1" + elif hlt2: + process = "hlt2" + elif spruce: + process = "spruce" + elif passthrough: + process = "pass" # Combine all lines and output in a global control flow. moore_control_node = moore_control_flow(options, streams, process, analytics) diff --git a/Hlt/Moore/python/Moore/lines.py b/Hlt/Moore/python/Moore/lines.py index 762387f20d9..f823502bdf2 100644 --- a/Hlt/Moore/python/Moore/lines.py +++ b/Hlt/Moore/python/Moore/lines.py @@ -378,6 +378,7 @@ class Hlt2Line(DecisionLine): extra_outputs (iterable of 2-tuple): List of `(name, DataHandle)` pairs. persistreco (bool): If True, request HLT2 reconstruction persistence. hlt1_filter_code (list(str)): string used to define a HLT1 filter. + stream (str): name of the stream to which the line belongs, defaults to None. Attributes: objects_to_persist (list of DataHandle): Objects which this lines @@ -412,6 +413,7 @@ class Hlt2Line(DecisionLine): algs, prescale=1.0, postscale=1.0, + stream=None, extra_outputs=None, persistreco=False, tagging_particles=False, @@ -461,12 +463,15 @@ class Hlt2Line(DecisionLine): else: algs += monitoring_algs super(Hlt2Line, self).__init__(name, algs, prescale, postscale) - if not self.name.startswith(self._CLASS_NAME_PREFIX): - raise ValueError( - "name {!r} does not start with {!r}".format( - name, self._CLASS_NAME_PREFIX + if not isinstance(self, SpruceLine): + if not self.name.startswith(self._CLASS_NAME_PREFIX): + raise ValueError( + "name {!r} does not start with {!r}".format( + name, self._CLASS_NAME_PREFIX + ) ) - ) + + self.extra_outputs = tuple(sorted(set(extra_outputs or []), key=hash)) self.raw_banks = tuple(sorted(set(raw_banks or []), key=hash)) self.persistreco = persistreco @@ -475,6 +480,7 @@ class Hlt2Line(DecisionLine): self.calo_clusters = calo_clusters self.pv_tracks = True if persistreco else pv_tracks self.track_ancestors = track_ancestors + self.stream = stream # The line guarantees that these objects will be present in the TES if # this line made a positive decision self.objects_to_persist = [] @@ -507,6 +513,7 @@ class Hlt2Line(DecisionLine): "track_ancestors": self.track_ancestors, "raw_banks": self.raw_banks, "produces_particles": is_output_producer, + "stream": self.stream, } def to_dict(self): @@ -732,6 +739,7 @@ class SpruceLine(Hlt2Line): algs, prescale=1.0, postscale=1.0, + stream=None, extra_outputs=None, raw_banks=None, persistreco=False, @@ -749,8 +757,7 @@ class SpruceLine(Hlt2Line): hlt2_filter = _return_filter( f"{name}_Hlt2Filter", self.hlt2_filter_code, - self._HLT2_FILTER_SOURCE_ID, - False if isinstance(self, PassLine) else True, + self._HLT2_FILTER_SOURCE_ID ) algs = [hlt2_filter] + algs @@ -761,6 +768,7 @@ class SpruceLine(Hlt2Line): algs=algs, prescale=prescale, postscale=postscale, + stream=stream, extra_outputs=extra_outputs, raw_banks=raw_banks, persistreco=persistreco, @@ -772,12 +780,7 @@ class SpruceLine(Hlt2Line): hlt1_filter_code=hlt1_filter_code, monitoring_variables=monitoring_variables, ) - if not self.name.startswith(self._CLASS_NAME_PREFIX): - raise ValueError( - "name {!r} does not start with {!r}".format( - name, self._CLASS_NAME_PREFIX - ) - ) + class PassLine(SpruceLine): diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index 8a0897d22b0..82c4b5edec9 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -600,9 +600,9 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): filter = f"{hlt2_linename}Decision" location = f"/Event/HLT2/{hlt2_linename}/Particles" - spruce_linename = hlt2_linename.replace("Hlt2", "SprucePass_Hlt2") + spruce_linename = hlt2_linename # print( - # f"Line {hlt2_linename} becomes {spruce_linename} with candidates at /Event/Spruce/{spruce_linename}/Particles" + # f"Line {hlt2_linename} becomes {spruce_linename} with candidates at /Event/Turbo/{spruce_linename}/Particles" # ) line_attributes = line_attributes[hlt2_linename] @@ -614,7 +614,7 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): algs = [] # Some lines do not output anything to `/Event/HLT2/{hlt2_linename}/Particles` if line_attributes["produces_particles"]: - # print(f"Line {hlt2_linename} produces location `/Event/HLT2/{hlt2_linename}/Particles`") + # print(f"Line {hlt2_linename} produces location `/Event/Turbo/{hlt2_linename}/Particles`") algs.append(hlt2_particles) else: print( @@ -642,5 +642,6 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): pv_tracks=line_attributes["pv_tracks"], track_ancestors=line_attributes["track_ancestors"], raw_banks=line_attributes["raw_banks"], + stream=line_attributes["stream"], ) return pass_spruceline -- GitLab From bf32ea425c3025b676fc6097f954e2e507268207 Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Tue, 4 Feb 2025 18:11:21 +0000 Subject: [PATCH 02/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50427831 --- .../options/sprucing/spruce_turbopass_overlap_check.py | 8 ++------ Hlt/Moore/python/Moore/LbExec.py | 5 ++++- Hlt/Moore/python/Moore/config.py | 4 ++-- Hlt/Moore/python/Moore/lines.py | 6 +----- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py index ae222e8adbe..dbe1e1c90c2 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_turbopass_overlap_check.py @@ -139,9 +139,7 @@ for ii in range(nevents): ## proto().track().ancestors().size() # Check CALO clusters - caloclusters = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][ - 0 - ].clusters() + caloclusters = TES["/Event/Turbo/HLT2/Rec/Calo/Electrons"][0].clusters() if not caloclusters or caloclusters.size() <= 0: raise RuntimeError("Check ERROR - Calo clusters not propagated") else: @@ -169,9 +167,7 @@ for ii in range(nevents): # Check extra_outputs for extraoutput in lineone_extraoutputs: - check_particles( - TES, f"/Event/Turbo/{lineone}/{extraoutput}" - ) + check_particles(TES, f"/Event/Turbo/{lineone}/{extraoutput}") # Check no overlap try: diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 68686336724..6953ff22d0a 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -65,7 +65,10 @@ class Options(DefaultOptions): persistreco_version.global_bind(version=self.persistreco_version) - if (self.process == ProcessTypes.Spruce or self.process == ProcessTypes.TurboSpruce): + if ( + self.process == ProcessTypes.Spruce + or self.process == ProcessTypes.TurboSpruce + ): reconstruction.global_bind(spruce=True, from_file=True) with super().apply_binds(): diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 7c806a551cc..b1ffa8ddc14 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -140,7 +140,7 @@ def moore_control_flow(options, streams, process, analytics=False): if process == "hlt2": event_output_prefix = reco_output_prefix = "/Event/HLT2" - if process =="spruce": + if process == "spruce": event_output_prefix = "/Event/Spruce" if getattr(options, "process", None): if options.process == "TurboSpruce": @@ -305,7 +305,7 @@ def run_moore( if getattr(options, "process", None): if options.process == "TurboSpruce": print("here we are") - process="spruce" + process = "spruce" else: assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( diff --git a/Hlt/Moore/python/Moore/lines.py b/Hlt/Moore/python/Moore/lines.py index f823502bdf2..a3338ac2e24 100644 --- a/Hlt/Moore/python/Moore/lines.py +++ b/Hlt/Moore/python/Moore/lines.py @@ -471,7 +471,6 @@ class Hlt2Line(DecisionLine): ) ) - self.extra_outputs = tuple(sorted(set(extra_outputs or []), key=hash)) self.raw_banks = tuple(sorted(set(raw_banks or []), key=hash)) self.persistreco = persistreco @@ -755,9 +754,7 @@ class SpruceLine(Hlt2Line): self.hlt2_filter_code = hlt2_filter_code if hlt2_filter_code: hlt2_filter = _return_filter( - f"{name}_Hlt2Filter", - self.hlt2_filter_code, - self._HLT2_FILTER_SOURCE_ID + f"{name}_Hlt2Filter", self.hlt2_filter_code, self._HLT2_FILTER_SOURCE_ID ) algs = [hlt2_filter] + algs @@ -782,7 +779,6 @@ class SpruceLine(Hlt2Line): ) - class PassLine(SpruceLine): """Object fully qualifying a Pass through line. -- GitLab From 72739662a198475134ce6bd905dcf8b9bd2da83e Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Tue, 4 Feb 2025 20:39:24 +0100 Subject: [PATCH 03/19] updates --- Hlt/Moore/python/Moore/LbExec.py | 13 +------------ Hlt/Moore/python/Moore/config.py | 8 ++++---- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 6953ff22d0a..f1d3580a31a 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -13,7 +13,7 @@ from enum import Enum from typing import Optional from GaudiConf.LbExec import Options as DefaultOptions -from GaudiConf.LbExec import TestOptionsBase +from GaudiConf.LbExec import ProcessTypes, TestOptionsBase from PyConf.application import ROOT_KEY from PyConf.packing import persistreco_version from PyConf.reading import reconstruction as reconstruction_reading @@ -22,20 +22,9 @@ from PyConf.reading import upfront_reconstruction as upfront_reconstruction_read from RecoConf.reconstruction_objects import reconstruction -class ProcessTypes(str, Enum): - Hlt1 = "Hlt1" - Hlt2 = "Hlt2" - TurboPass = "TurboPass" - Spruce = "Spruce" - TurboSpruce = "TurboSpruce" - ReconstructionOnly = "ReconstructionOnly" - Monitoring = "Monitoring" - UserDefined = "UserDefined" - class Options(DefaultOptions): tck: Optional[int] = None - process: ProcessTypes = ProcessTypes.Hlt2 @contextmanager def apply_binds(self): diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index b1ffa8ddc14..a1f90159076 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -140,15 +140,15 @@ def moore_control_flow(options, streams, process, analytics=False): if process == "hlt2": event_output_prefix = reco_output_prefix = "/Event/HLT2" - if process == "spruce": + if process =="spruce": event_output_prefix = "/Event/Spruce" if getattr(options, "process", None): if options.process == "TurboSpruce": event_output_prefix = "/Event/Turbo" print("event_output_prefix", event_output_prefix) - reco_output_prefix = "/Event/HLT2" - ## The next bit is indented!!!! + reco_output_prefix = "/Event/HLT2" + ## The next bit is indented (line_output_cf, extra_locations_to_persist, packed_data) = ( persist_line_outputs( streams=streams, @@ -305,7 +305,7 @@ def run_moore( if getattr(options, "process", None): if options.process == "TurboSpruce": print("here we are") - process = "spruce" + process="spruce" else: assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( -- GitLab From da41cdfc4e4654f4759b9728c7e74faaf7e61fe5 Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Tue, 4 Feb 2025 19:40:04 +0000 Subject: [PATCH 04/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50434777 --- Hlt/Moore/python/Moore/LbExec.py | 1 - Hlt/Moore/python/Moore/config.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index f1d3580a31a..3f0a31cb379 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -22,7 +22,6 @@ from PyConf.reading import upfront_reconstruction as upfront_reconstruction_read from RecoConf.reconstruction_objects import reconstruction - class Options(DefaultOptions): tck: Optional[int] = None diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index a1f90159076..72111414326 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -140,7 +140,7 @@ def moore_control_flow(options, streams, process, analytics=False): if process == "hlt2": event_output_prefix = reco_output_prefix = "/Event/HLT2" - if process =="spruce": + if process == "spruce": event_output_prefix = "/Event/Spruce" if getattr(options, "process", None): if options.process == "TurboSpruce": @@ -305,7 +305,7 @@ def run_moore( if getattr(options, "process", None): if options.process == "TurboSpruce": print("here we are") - process="spruce" + process = "spruce" else: assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( -- GitLab From e079b87b5bacf1558fe3af8ee610b5c4f9f119ba Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Wed, 5 Feb 2025 17:50:59 +0100 Subject: [PATCH 05/19] provide list of streams to lines --- .../sprucing/lbexec_yamls/spruce_overlap.yaml | 2 +- .../python/Hlt2Conf/Sprucing_tests.py | 32 ++++++++----- Hlt/Moore/python/Moore/LbExec.py | 1 + Hlt/Moore/python/Moore/production.py | 45 +++++++++---------- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml index 6634a40e5d6..d68563977f1 100644 --- a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml +++ b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml @@ -9,7 +9,7 @@ input_process: Hlt2 evt_max: -1 geometry_version: run3/trunk conditions_version: jonrob/all-pmts-active - +output_streams_config: "/eos/lhcb/wg/dpa/wp1/fortests/test_streaming_config.json" process: TurboSpruce input_raw_format : 0.5 diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 75402d09f33..090b1b5407b 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -303,6 +303,7 @@ def turbo_overlap(options: Options): def spruce_overlap(options: Options): + import json """ Test for the technical overlap of passthrough streams created in the sprucing-streaming step @@ -314,22 +315,31 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ + stream="default" input_line_config = options.input_streams_attributes_file - - spruce_streams = { - "streamone": ["Hlt2.*one.*"], - "streamtwo": ["Hlt2.*two.*"], - "streamthree": ["Hlt2.*three.*"], - } - + input_stream_config = options.output_streams_config + + with open (input_stream_config, "r") as json_file: + streaming_config = json.load(json_file) + with open (input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] + + # Merge the line_config and the streaming_config + for line, l_config in line_config.items(): + for stream, s_config in streaming_config.items(): + if line in s_config: + l_config["stream"] = stream + break + else: + raise Exception(f"Line {line} not found in any stream") + + print(f"line_config: {line_config}") custom_prescales = {"Hlt2Linetwo": 0.5} def make_streams(): return turbo_spruce( - input_line_config, - spruce_streams, - custom_prescales=custom_prescales, - stream="default", + line_config, + custom_prescales=custom_prescales ) with list_of_full_stream_lines.bind(lines=[]): diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 3f0a31cb379..2a48bf58762 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -24,6 +24,7 @@ from RecoConf.reconstruction_objects import reconstruction class Options(DefaultOptions): tck: Optional[int] = None + output_streams_config: Optional[str] = None @contextmanager def apply_binds(self): diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index 82c4b5edec9..0de3fd6d258 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -520,57 +520,53 @@ def _spruce( def turbo_spruce( - input_line_config, spruce_streams, custom_prescales={}, stream="turbo" + line_config, custom_prescales={} ): """ - Creates and configures sprucing streams for the passthrough of HLT2 TURBO output. + Creates and configures sprucing streams for HLT2 TURBO output. - This function reads the attributes of all Hlt2Lines - from a JSON file made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their attributes and - the given regex patterns in `spruce_streams`. If required it can apply custom prescales. + This function reads the attributes of Hlt2Lines from a dict made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their "stream" attribute. If required it can apply custom prescales. Args: - input_line_config (str): JSON file made at the HLT2 stage with stream and line attributes - spruce_streams (dict): A dictionary mapping stream names to regex - pattern lists for Sprucing streams. + line_config (dict): dict made at the HLT2 stage with line attributes custom_prescales (dict, optional): A dictionary mapping line names to prescale factors. Defaults to empty dictionary. - stream (str, optional): The stream to be configured. Defaults to "turbo". Returns: Streams: An object containing the configured sprucing streams for passthrough of TURBO output. """ - with open(input_line_config, "r") as json_file: - line_attributes = json.load(json_file)[stream] - turbo_lines = list(line_attributes.keys()) + turbo_lines = list(line_config.keys()) # print(f"Lines in HLT2 stream requested: {turbo_lines}") - assert all( - isinstance(regex_list, list) for regex_list in spruce_streams.values() - ), "The `spruce_streams` dict must map stream names to regex pattern LISTS" + list_of_streams=set(line_config[line]["stream"] for line in line_config.keys()) - # Can stream lines according to regex + ### Check that all lines have a stream + if None in list_of_streams: + raise ValueError("At least one line has no stream attribute assigned!!!") + print(f"Streams found in line attributes file are : {list_of_streams}") + + # Streaming is done via exact "regex" list of lines streaming = { stream_name: [ line for line in turbo_lines - if any(re.match(regex, line) for regex in regex_list) + if line_config[line]["stream"]==stream_name ] - for stream_name, regex_list in spruce_streams.items() + for stream_name in list_of_streams } - # for stream_name in streaming.keys(): - # print( - # f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") + for stream_name in streaming.keys(): + print( + f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") streams = [ Stream( stream_name, lines=[ _make_pass_spruceline( - line_attributes, line, custom_prescales=custom_prescales + line_config, line, custom_prescales=custom_prescales ) for line in streaming[stream_name] ], @@ -589,7 +585,7 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): Create a SpruceLine for a given Hlt2Line. Args: - line_attributes (dict): Attributes for all lines eg. `json[stream]` where json is read from `options.input_streams_attributes_file`. + line_attributes (dict): Attributes for all lines. hlt2_linename (str): The name of the Hlt2Line to be converted into a SpruceLine. custom_prescales (dict, optional): A dictionary mapping lines to custom prescales. Defaults to empty dictionary. @@ -607,7 +603,6 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): line_attributes = line_attributes[hlt2_linename] line_attributes["prescale"] = custom_prescales.get(hlt2_linename, 1.0) - # print(line_attributes) with upfront_decoder.bind(source="Hlt2"): hlt2_particles = get_particles(location) @@ -618,7 +613,7 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): algs.append(hlt2_particles) else: print( - f"Line {hlt2_linename} does not produce location `/Event/HLT2/{hlt2_linename}/Particles`, skipping this location" + f"Line {hlt2_linename} did not produce location `/Event/HLT2/{hlt2_linename}/Particles`, skipping this location" ) extra_outputs_tuplelist = [] -- GitLab From a6e04b7ff5433b11b053e68613e4f98ed476e173 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 14:39:12 +0100 Subject: [PATCH 06/19] working example --- .../python/Hlt2Conf/Sprucing_tests.py | 11 ++- .../Sprucing_production_pp.py | 4 +- Hlt/Moore/python/Moore/production.py | 76 +++++++++++-------- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 090b1b5407b..b914b5952b8 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -317,14 +317,14 @@ def spruce_overlap(options: Options): stream="default" input_line_config = options.input_streams_attributes_file - input_stream_config = options.output_streams_config + stream_config = options.output_streams_config - with open (input_stream_config, "r") as json_file: + # Open and merge the line_config and the streaming_config + with open (stream_config, "r") as json_file: streaming_config = json.load(json_file) with open (input_line_config, "r") as json_file: line_config = json.load(json_file)[stream] - # Merge the line_config and the streaming_config for line, l_config in line_config.items(): for stream, s_config in streaming_config.items(): if line in s_config: @@ -333,13 +333,16 @@ def spruce_overlap(options: Options): else: raise Exception(f"Line {line} not found in any stream") + assert(sum(map(len, streaming_config.values()))==len(line_config.keys())), f"There are {len(line_config.keys())} lines to be run but only {sum(map(len, streaming_config.values()))} lines in the streams config" + + print(f"line_config: {line_config}") custom_prescales = {"Hlt2Linetwo": 0.5} def make_streams(): return turbo_spruce( line_config, - custom_prescales=custom_prescales + custom_prescales=custom_prescales, use_regex=False ) with list_of_full_stream_lines.bind(lines=[]): diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py index 6e15452dac9..7ca5d3290f3 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py @@ -158,11 +158,9 @@ from Moore.production import turbo_spruce def make_turbo_spruce_prod_streams(input_streams_attributes_file): - # create_or_reuse_rootIOAlg(options) - # custom_prescales = {"Hlt2Linetwo": 0.5} - return turbo_spruce(input_streams_attributes_file, turbolinedict) + return turbo_spruce(input_streams_attributes_file, use_regex=True, spruce_streams=turbolinedict) def turbo_spruce_production(options: Options): diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index 0de3fd6d258..d4dd456edc1 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -520,53 +520,71 @@ def _spruce( def turbo_spruce( - line_config, custom_prescales={} + line_config, custom_prescales={}, use_regex=True, spruce_streams=None ): """ Creates and configures sprucing streams for HLT2 TURBO output. - This function reads the attributes of Hlt2Lines from a dict made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their "stream" attribute. If required it can apply custom prescales. + This function reads the attributes of Hlt2Lines from a dict made at the HLT2 stage. It streams these Hlt2Lines as SpruceLines based on their "stream" attribute OR by regex. If required it can apply custom prescales. Args: - line_config (dict): dict made at the HLT2 stage with line attributes + line_config (dict): dict made at the HLT2 stage with line attributes. This dict must have the following structure: {"line_name": {"name": "line_name", "stream": "stream_name", "prescale": 1.0} ...}. See Sprucing_tests:spruce_overlap for an example custom_prescales (dict, optional): A dictionary mapping line names to prescale factors. Defaults to empty dictionary. + use_regex (bool, optional): If True, stream lines according to spruce_streams regex. Defaults to True. + spruce_streams (dict, optional): A dictionary mapping stream names to regex patterns. Defaults to None. Returns: Streams: An object containing the configured sprucing streams for passthrough of TURBO output. """ - - turbo_lines = list(line_config.keys()) # print(f"Lines in HLT2 stream requested: {turbo_lines}") - list_of_streams=set(line_config[line]["stream"] for line in line_config.keys()) - - ### Check that all lines have a stream - if None in list_of_streams: - raise ValueError("At least one line has no stream attribute assigned!!!") - print(f"Streams found in line attributes file are : {list_of_streams}") - - # Streaming is done via exact "regex" list of lines - streaming = { + if use_regex: + assert all( + isinstance(regex_list, list) for regex_list in spruce_streams.values() + ), "The `spruce_streams` dict for use_regex=True must map stream names to regex pattern LISTS" + # Can stream lines according to regex + streaming = { stream_name: [ line for line in turbo_lines - if line_config[line]["stream"]==stream_name + if any(re.match(regex, line) for regex in regex_list) ] - for stream_name in list_of_streams + for stream_name, regex_list in spruce_streams.items() } + else: #Using stream attribute in line_config to stream lines + list_of_streams=set(line_config[line]["stream"] for line in line_config.keys()) + + ### Check that all lines have a stream + if None in list_of_streams: + raise ValueError("At least one line has no stream attribute assigned!!!") + print(f"Streams found in line attributes file are : {list_of_streams}") + + # Streaming is done via exact "regex" list of lines + streaming = { + stream_name: [ + line + for line in turbo_lines + if line_config[line].get("stream")==stream_name + ] + for stream_name in list_of_streams + } + for stream_name in streaming.keys(): print( f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") + assert(sum(map(len, streaming.values()))==len(line_config.keys())), f"There are {len(line_config.keys())} lines to be streamed but only {sum(map(len, streaming.values()))} lines in the streams config" + + streams = [ Stream( stream_name, lines=[ _make_pass_spruceline( - line_config, line, custom_prescales=custom_prescales + line_config[line], custom_prescales=custom_prescales ) for line in streaming[stream_name] ], @@ -580,13 +598,12 @@ def turbo_spruce( return my_streams -def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): +def _make_pass_spruceline(line_attributes, custom_prescales={}): """ Create a SpruceLine for a given Hlt2Line. Args: - line_attributes (dict): Attributes for all lines. - hlt2_linename (str): The name of the Hlt2Line to be converted into a SpruceLine. + line_attributes (dict): Attributes for line. custom_prescales (dict, optional): A dictionary mapping lines to custom prescales. Defaults to empty dictionary. Returns: @@ -594,26 +611,25 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): """ from PyConf.reading import get_particles, upfront_decoder - filter = f"{hlt2_linename}Decision" - location = f"/Event/HLT2/{hlt2_linename}/Particles" - spruce_linename = hlt2_linename + linename=line_attributes["name"] + filter = f"{linename}Decision" + location = f"/Event/HLT2/{linename}/Particles" + # print( - # f"Line {hlt2_linename} becomes {spruce_linename} with candidates at /Event/Turbo/{spruce_linename}/Particles" + # f"Line {linename} produces candidates at /Event/Turbo/{spruce_linename}/Particles" # ) - line_attributes = line_attributes[hlt2_linename] - line_attributes["prescale"] = custom_prescales.get(hlt2_linename, 1.0) + line_attributes["prescale"] = custom_prescales.get(linename, 1.0) with upfront_decoder.bind(source="Hlt2"): hlt2_particles = get_particles(location) algs = [] # Some lines do not output anything to `/Event/HLT2/{hlt2_linename}/Particles` if line_attributes["produces_particles"]: - # print(f"Line {hlt2_linename} produces location `/Event/Turbo/{hlt2_linename}/Particles`") algs.append(hlt2_particles) else: print( - f"Line {hlt2_linename} did not produce location `/Event/HLT2/{hlt2_linename}/Particles`, skipping this location" + f"Line {linename} did not produce location `/Event/HLT2/{linename}/Particles`, skipping this location" ) extra_outputs_tuplelist = [] @@ -622,10 +638,10 @@ def _make_pass_spruceline(line_attributes, hlt2_linename, custom_prescales={}): for loc in line_attributes["extra_outputs"]: # print(f"Adding extra_output {loc} for line {spruce_linename}") extra_outputs_tuplelist.append( - (loc, get_particles(f"/Event/HLT2/{hlt2_linename}/{loc}/Particles")) + (loc, get_particles(f"/Event/HLT2/{linename}/{loc}/Particles")) ) pass_spruceline = SpruceLine( - name=spruce_linename, + name=linename, hlt2_filter_code=filter, algs=algs, extra_outputs=extra_outputs_tuplelist, -- GitLab From ccc40082de3dcec1c092fe96449fd4389e4a8671 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 15:03:54 +0100 Subject: [PATCH 07/19] remove prints --- Hlt/Moore/python/Moore/LbExec.py | 3 ++- Hlt/Moore/python/Moore/config.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 2a48bf58762..e71330d8c4e 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -12,7 +12,7 @@ from contextlib import contextmanager from enum import Enum from typing import Optional -from GaudiConf.LbExec import Options as DefaultOptions +from GaudiConf.LbExec import Options as DefaultOptions, ProcessTypes from GaudiConf.LbExec import ProcessTypes, TestOptionsBase from PyConf.application import ROOT_KEY from PyConf.packing import persistreco_version @@ -25,6 +25,7 @@ from RecoConf.reconstruction_objects import reconstruction class Options(DefaultOptions): tck: Optional[int] = None output_streams_config: Optional[str] = None + process: Optional[ProcessTypes] = None @contextmanager def apply_binds(self): diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 72111414326..3c419ffc2b6 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -145,7 +145,6 @@ def moore_control_flow(options, streams, process, analytics=False): if getattr(options, "process", None): if options.process == "TurboSpruce": event_output_prefix = "/Event/Turbo" - print("event_output_prefix", event_output_prefix) reco_output_prefix = "/Event/HLT2" ## The next bit is indented @@ -304,7 +303,6 @@ def run_moore( if getattr(options, "process", None): if options.process == "TurboSpruce": - print("here we are") process = "spruce" else: -- GitLab From b69b51cbcc9befe9628d3971e12cb50a2bd6b52c Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Thu, 6 Feb 2025 14:04:47 +0000 Subject: [PATCH 08/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50547542 --- .../python/Hlt2Conf/Sprucing_tests.py | 15 +++---- .../Sprucing_production_pp.py | 6 +-- Hlt/Moore/python/Moore/LbExec.py | 2 +- Hlt/Moore/python/Moore/production.py | 42 ++++++++++--------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index b914b5952b8..d08fe094695 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -304,6 +304,7 @@ def turbo_overlap(options: Options): def spruce_overlap(options: Options): import json + """ Test for the technical overlap of passthrough streams created in the sprucing-streaming step @@ -315,14 +316,14 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ - stream="default" + stream = "default" input_line_config = options.input_streams_attributes_file stream_config = options.output_streams_config # Open and merge the line_config and the streaming_config - with open (stream_config, "r") as json_file: + with open(stream_config, "r") as json_file: streaming_config = json.load(json_file) - with open (input_line_config, "r") as json_file: + with open(input_line_config, "r") as json_file: line_config = json.load(json_file)[stream] for line, l_config in line_config.items(): @@ -333,16 +334,16 @@ def spruce_overlap(options: Options): else: raise Exception(f"Line {line} not found in any stream") - assert(sum(map(len, streaming_config.values()))==len(line_config.keys())), f"There are {len(line_config.keys())} lines to be run but only {sum(map(len, streaming_config.values()))} lines in the streams config" - + assert sum(map(len, streaming_config.values())) == len(line_config.keys()), ( + f"There are {len(line_config.keys())} lines to be run but only {sum(map(len, streaming_config.values()))} lines in the streams config" + ) print(f"line_config: {line_config}") custom_prescales = {"Hlt2Linetwo": 0.5} def make_streams(): return turbo_spruce( - line_config, - custom_prescales=custom_prescales, use_regex=False + line_config, custom_prescales=custom_prescales, use_regex=False ) with list_of_full_stream_lines.bind(lines=[]): diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py index 7ca5d3290f3..bddb9e7dfe6 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py @@ -158,9 +158,9 @@ from Moore.production import turbo_spruce def make_turbo_spruce_prod_streams(input_streams_attributes_file): - - - return turbo_spruce(input_streams_attributes_file, use_regex=True, spruce_streams=turbolinedict) + return turbo_spruce( + input_streams_attributes_file, use_regex=True, spruce_streams=turbolinedict + ) def turbo_spruce_production(options: Options): diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index e71330d8c4e..b4578641626 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -12,7 +12,7 @@ from contextlib import contextmanager from enum import Enum from typing import Optional -from GaudiConf.LbExec import Options as DefaultOptions, ProcessTypes +from GaudiConf.LbExec import Options as DefaultOptions from GaudiConf.LbExec import ProcessTypes, TestOptionsBase from PyConf.application import ROOT_KEY from PyConf.packing import persistreco_version diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index d4dd456edc1..e49c28f43f1 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -519,9 +519,7 @@ def _spruce( return config -def turbo_spruce( - line_config, custom_prescales={}, use_regex=True, spruce_streams=None -): +def turbo_spruce(line_config, custom_prescales={}, use_regex=True, spruce_streams=None): """ Creates and configures sprucing streams for HLT2 TURBO output. @@ -542,20 +540,24 @@ def turbo_spruce( if use_regex: assert all( - isinstance(regex_list, list) for regex_list in spruce_streams.values() - ), "The `spruce_streams` dict for use_regex=True must map stream names to regex pattern LISTS" + isinstance(regex_list, list) for regex_list in spruce_streams.values() + ), ( + "The `spruce_streams` dict for use_regex=True must map stream names to regex pattern LISTS" + ) # Can stream lines according to regex streaming = { - stream_name: [ - line - for line in turbo_lines - if any(re.match(regex, line) for regex in regex_list) - ] - for stream_name, regex_list in spruce_streams.items() - } + stream_name: [ + line + for line in turbo_lines + if any(re.match(regex, line) for regex in regex_list) + ] + for stream_name, regex_list in spruce_streams.items() + } - else: #Using stream attribute in line_config to stream lines - list_of_streams=set(line_config[line]["stream"] for line in line_config.keys()) + else: # Using stream attribute in line_config to stream lines + list_of_streams = set( + line_config[line]["stream"] for line in line_config.keys() + ) ### Check that all lines have a stream if None in list_of_streams: @@ -567,17 +569,17 @@ def turbo_spruce( stream_name: [ line for line in turbo_lines - if line_config[line].get("stream")==stream_name + if line_config[line].get("stream") == stream_name ] for stream_name in list_of_streams } for stream_name in streaming.keys(): - print( - f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") - - assert(sum(map(len, streaming.values()))==len(line_config.keys())), f"There are {len(line_config.keys())} lines to be streamed but only {sum(map(len, streaming.values()))} lines in the streams config" + print(f"Lines in {stream_name} to be spruced: {streaming[stream_name]}") + assert sum(map(len, streaming.values())) == len(line_config.keys()), ( + f"There are {len(line_config.keys())} lines to be streamed but only {sum(map(len, streaming.values()))} lines in the streams config" + ) streams = [ Stream( @@ -611,7 +613,7 @@ def _make_pass_spruceline(line_attributes, custom_prescales={}): """ from PyConf.reading import get_particles, upfront_decoder - linename=line_attributes["name"] + linename = line_attributes["name"] filter = f"{linename}Decision" location = f"/Event/HLT2/{linename}/Particles" -- GitLab From 63b49eddf37c004b399c5e240acb7325a4a555b7 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 18:34:01 +0100 Subject: [PATCH 09/19] bw tests and rm some tests --- .../Sprucing_production_pp.py | 13 ++++-- .../spruce_bandwidth_turbo_streams.py | 8 +++- ...test_excl_spruce_2022_data_bandq_check.qmt | 42 ----------------- .../test_excl_spruce_2022_data_qee_check.qmt | 42 ----------------- .../test_excl_spruce_2022_data_sl_check.qmt | 42 ----------------- .../test_pass_spruce_2022_data_b2cc_check.qmt | 42 ----------------- ...test_pass_spruce_2022_data_bandq_check.qmt | 42 ----------------- .../test_pass_spruce_2022_data_bnoc_check.qmt | 42 ----------------- ...test_pass_spruce_2022_data_charm_check.qmt | 42 ----------------- .../test_pass_spruce_2022_data_qee_check.qmt | 42 ----------------- .../test_pass_spruce_2022_data_sl_check.qmt | 42 ----------------- ...rcal_spruce_2022_data_monitoring_check.qmt | 45 ------------------ ...l_spruce_2022_data_monitoring_rb_check.qmt | 46 ------------------- ...turcal_spruce_2022_data_trackeff_check.qmt | 45 ------------------ ...cal_spruce_2022_data_trackeff_rb_check.qmt | 46 ------------------- Hlt/Moore/python/Moore/LbExec.py | 1 - 16 files changed, 17 insertions(+), 565 deletions(-) delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py index bddb9e7dfe6..1d1a1e85461 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py @@ -157,15 +157,22 @@ turbolinedict = { from Moore.production import turbo_spruce -def make_turbo_spruce_prod_streams(input_streams_attributes_file): +def make_turbo_spruce_prod_streams(line_config): return turbo_spruce( - input_streams_attributes_file, use_regex=True, spruce_streams=turbolinedict + line_config, use_regex=True, spruce_streams=turbolinedict ) def turbo_spruce_production(options: Options): + import json + + input_line_config = options.input_streams_attributes_file + stream="turbo" + with open(input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] + def make_streams(): - return make_turbo_spruce_prod_streams(options.input_streams_attributes_file) + return make_turbo_spruce_prod_streams(line_config) config = run_moore(options, make_streams, public_tools=[]) return config diff --git a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py index 0f7edb7d21c..7f43160d7a7 100644 --- a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py +++ b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py @@ -27,6 +27,7 @@ from RecoConf.reconstruction_objects import reconstruction log = logging.getLogger() options.input_process = "Hlt2" +options.process="TurboSpruce" fname_helper = FileNameHelper(process="spruce", stream_config="turbo") fname_helper.make_tmp_dirs() @@ -38,8 +39,13 @@ options.input_streams_attributes_file = FileNameHelper( ).streams_attributes_file() +input_line_config = options.input_streams_attributes_file +stream="turbo" +with open(input_line_config, "r") as json_file: + line_config = json.load(json_file)[stream] + def make_streams(): - real_streams = make_turbo_spruce_prod_streams(options.input_streams_attributes_file) + real_streams = make_turbo_spruce_prod_streams(line_config) # Write out stream configuration to JSON file for use later in the test with open(fname_helper.stream_config_json_path(), "w") as f: diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt deleted file mode 100644 index 4da1e673aac..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_bandq_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.bandq.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - bandq - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt deleted file mode 100644 index 1a34f569169..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_qee_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.qee.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - qee - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt deleted file mode 100644 index e2b670a9009..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_excl_spruce_2022_data_sl_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_excl_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - spruce_all_lines_production.sl.dst - -manifest - spruce_all_lines_production.tck.json - -job_type - excl - -stream - sl - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt deleted file mode 100644 index 68da553c431..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_b2cc_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.b2cc.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - b2cc - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt deleted file mode 100644 index 24c9e0716ee..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bandq_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.bandq.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - bandq - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt deleted file mode 100644 index 68ae8fd7fa2..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_bnoc_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.bnoc.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - bnoc - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt deleted file mode 100644 index f9343c58cb3..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_charm_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.charm.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - charm - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt deleted file mode 100644 index 08efc243515..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_qee_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.qee.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - qee - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt deleted file mode 100644 index 8578829b0de..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_pass_spruce_2022_data_sl_check.qmt +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - sprucing.test_pass_spruce_2022_dataPASS - - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_all_lines_production.sl.dst - -manifest - pass_all_lines_production.tck.json - -job_type - pass - -stream - sl - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt deleted file mode 100644 index 16ff0c69be2..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_check.qmt +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.monitoring.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - monitoring - -rb_to_check - 16 - -rb_to_check_not - 73649138477 - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt deleted file mode 100644 index 1fd0294f911..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_monitoring_rb_check.qmt +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.monitoring_raw.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - monitoring_raw - -rb_to_check - 1673649138477 - -true -../refs/test_turcal_spruce_2022_data_monitoring_rb_check.ref - - -from Moore.qmtest.exclusions import remove_known_warnings -from GaudiConf.QMTest.LHCbExclusions import preprocessor -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 1}, - stdout=remove_known_warnings(stdout)) -validateWithReference(preproc=preprocessor) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt deleted file mode 100644 index 1c3e89f9a18..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_check.qmt +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.trackeff.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - trackeff - -rb_to_check - 16 - -rb_to_check_not - 73649138477 - -true - - -from Moore.qmtest.exclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, - stdout=remove_known_warnings(stdout)) - - - diff --git a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt b/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt deleted file mode 100644 index d413f48ade6..00000000000 --- a/Hlt/Hlt2Conf/tests/qmtest/sprucing.qms/test_turcal_spruce_2022_data_trackeff_rb_check.qmt +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - sprucing.test_turcal_spruce_2022_dataPASS - -python -300 - - $HLT2CONFROOT/tests/options/sprucing/spruce_check_2022_data.py - -input - pass_turcal_lines_production.trackeff_raw.dst - -manifest - pass_turcal_lines_production.tck.json - -job_type - pass_turcal - -stream - trackeff_raw - -rb_to_check - 1673649138477 - -true -../refs/test_turcal_spruce_2022_data_trackeff_rb_check.ref - - -from Moore.qmtest.exclusions import remove_known_warnings -from GaudiConf.QMTest.LHCbExclusions import preprocessor -countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 1}, - stdout=remove_known_warnings(stdout)) -validateWithReference(preproc=preprocessor) - - - diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index b4578641626..2a48bf58762 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -25,7 +25,6 @@ from RecoConf.reconstruction_objects import reconstruction class Options(DefaultOptions): tck: Optional[int] = None output_streams_config: Optional[str] = None - process: Optional[ProcessTypes] = None @contextmanager def apply_binds(self): -- GitLab From 3b8ce0c57f2a26d854ada547af78740ae261edb9 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 19:52:01 +0100 Subject: [PATCH 10/19] use and --- Hlt/Moore/python/Moore/config.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 3c419ffc2b6..56355479011 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -142,9 +142,8 @@ def moore_control_flow(options, streams, process, analytics=False): if process == "spruce": event_output_prefix = "/Event/Spruce" - if getattr(options, "process", None): - if options.process == "TurboSpruce": - event_output_prefix = "/Event/Turbo" + if getattr(options, "process", None) and options.process == "TurboSpruce": + event_output_prefix = "/Event/Turbo" reco_output_prefix = "/Event/HLT2" ## The next bit is indented @@ -301,8 +300,7 @@ def run_moore( options.output_streams_attributes_file, streams ) - if getattr(options, "process", None): - if options.process == "TurboSpruce": + if getattr(options, "process", None) and options.process == "TurboSpruce": process = "spruce" else: -- GitLab From f1dbad1c94451c7fc06c13f455dbfb8716c029f4 Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Thu, 6 Feb 2025 18:52:46 +0000 Subject: [PATCH 11/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50566005 --- .../Hlt2Conf/sprucing_settings/Sprucing_production_pp.py | 6 ++---- .../options/bandwidth/spruce_bandwidth_turbo_streams.py | 5 +++-- Hlt/Moore/python/Moore/config.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py index 1d1a1e85461..4399f24bc76 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/sprucing_settings/Sprucing_production_pp.py @@ -158,16 +158,14 @@ from Moore.production import turbo_spruce def make_turbo_spruce_prod_streams(line_config): - return turbo_spruce( - line_config, use_regex=True, spruce_streams=turbolinedict - ) + return turbo_spruce(line_config, use_regex=True, spruce_streams=turbolinedict) def turbo_spruce_production(options: Options): import json input_line_config = options.input_streams_attributes_file - stream="turbo" + stream = "turbo" with open(input_line_config, "r") as json_file: line_config = json.load(json_file)[stream] diff --git a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py index 7f43160d7a7..13a0a863429 100644 --- a/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py +++ b/Hlt/Hlt2Conf/tests/options/bandwidth/spruce_bandwidth_turbo_streams.py @@ -27,7 +27,7 @@ from RecoConf.reconstruction_objects import reconstruction log = logging.getLogger() options.input_process = "Hlt2" -options.process="TurboSpruce" +options.process = "TurboSpruce" fname_helper = FileNameHelper(process="spruce", stream_config="turbo") fname_helper.make_tmp_dirs() @@ -40,10 +40,11 @@ options.input_streams_attributes_file = FileNameHelper( input_line_config = options.input_streams_attributes_file -stream="turbo" +stream = "turbo" with open(input_line_config, "r") as json_file: line_config = json.load(json_file)[stream] + def make_streams(): real_streams = make_turbo_spruce_prod_streams(line_config) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 56355479011..f79b3325b7e 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -301,7 +301,7 @@ def run_moore( ) if getattr(options, "process", None) and options.process == "TurboSpruce": - process = "spruce" + process = "spruce" else: assert hlt1 ^ hlt2 ^ spruce ^ passthrough, ( -- GitLab From a32f572d818d32ea8cc82cbd42c7903c6b3c21fa Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 22:29:01 +0100 Subject: [PATCH 12/19] tests --- Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml | 2 +- .../tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref | 1 + Hlt/Moore/python/Moore/LbExec.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml index d68563977f1..925af306520 100644 --- a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml +++ b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml @@ -9,7 +9,7 @@ input_process: Hlt2 evt_max: -1 geometry_version: run3/trunk conditions_version: jonrob/all-pmts-active -output_streams_config: "/eos/lhcb/wg/dpa/wp1/fortests/test_streaming_config.json" +output_streams_config: "root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/fortests/test_streaming_config.json" process: TurboSpruce input_raw_format : 0.5 diff --git a/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref b/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref index 724e6d82bf4..afdca682344 100644 --- a/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref +++ b/Hlt/Hlt2Conf/tests/refs/test_turcal_spruce_2022_data_pid_rb_check.ref @@ -49,6 +49,7 @@ rb_to_check_not None |-phoenix_filename = '' (default: '') |-preamble_algs = [] (default: []) |-print_freq = 10000 (default: 10000) +|-process = None |-python_logging_level = 20 (default: 20) |-require_specific_decoding_keys = [] (default: []) |-root_ioalg_name = 'RootIOAlg' (default: 'RootIOAlg') diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 2a48bf58762..401139a6c7a 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -57,6 +57,7 @@ class Options(DefaultOptions): if ( self.process == ProcessTypes.Spruce or self.process == ProcessTypes.TurboSpruce + or self.process == ProcessTypes.TurboPass ): reconstruction.global_bind(spruce=True, from_file=True) -- GitLab From ffea8b3f8b81113d80b2b49aafb4f50e8c265dc7 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Thu, 6 Feb 2025 22:55:30 +0100 Subject: [PATCH 13/19] move josn --- .../options/sprucing/lbexec_yamls/spruce_overlap.yaml | 1 - Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py | 7 ++++++- Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json | 1 + Hlt/Moore/python/Moore/LbExec.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json diff --git a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml index 925af306520..db386a2b248 100644 --- a/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml +++ b/Hlt/Hlt2Conf/options/sprucing/lbexec_yamls/spruce_overlap.yaml @@ -9,7 +9,6 @@ input_process: Hlt2 evt_max: -1 geometry_version: run3/trunk conditions_version: jonrob/all-pmts-active -output_streams_config: "root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/fortests/test_streaming_config.json" process: TurboSpruce input_raw_format : 0.5 diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index d08fe094695..7901e7745ab 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -316,9 +316,14 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ + import json + from pathlib import Path + turbolinedict = (Path(__file__).parent / "test_streaming_config.json") + stream = "default" input_line_config = options.input_streams_attributes_file - stream_config = options.output_streams_config + #stream_config = options.output_streams_config + stream_config = turbolinedict # Open and merge the line_config and the streaming_config with open(stream_config, "r") as json_file: diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json new file mode 100644 index 00000000000..607e25acfc8 --- /dev/null +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json @@ -0,0 +1 @@ +{"streamone": ["Hlt2Lineone_extraoutputs"], "streamtwo": ["Hlt2Linetwo"], "streamthree": ["Hlt2Linethree"]} \ No newline at end of file diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 401139a6c7a..03948693a5c 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -24,7 +24,7 @@ from RecoConf.reconstruction_objects import reconstruction class Options(DefaultOptions): tck: Optional[int] = None - output_streams_config: Optional[str] = None + output_streams_config: Optional[str] = "test_streaming_config.json" @contextmanager def apply_binds(self): -- GitLab From 420173361ed65f795e420f0675872753043ed968 Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Thu, 6 Feb 2025 21:56:15 +0000 Subject: [PATCH 14/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50571051 --- Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py | 6 +++--- Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 7901e7745ab..9a8573cb168 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -316,13 +316,13 @@ def spruce_overlap(options: Options): This test shows how to remove the overlap using SpruceLine based on the output of hlt2_foroverlapcheck.py """ - import json from pathlib import Path - turbolinedict = (Path(__file__).parent / "test_streaming_config.json") + + turbolinedict = Path(__file__).parent / "test_streaming_config.json" stream = "default" input_line_config = options.input_streams_attributes_file - #stream_config = options.output_streams_config + # stream_config = options.output_streams_config stream_config = turbolinedict # Open and merge the line_config and the streaming_config diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json index 607e25acfc8..8a61b1beed0 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/test_streaming_config.json @@ -1 +1 @@ -{"streamone": ["Hlt2Lineone_extraoutputs"], "streamtwo": ["Hlt2Linetwo"], "streamthree": ["Hlt2Linethree"]} \ No newline at end of file +{"streamone": ["Hlt2Lineone_extraoutputs"], "streamtwo": ["Hlt2Linetwo"], "streamthree": ["Hlt2Linethree"]} -- GitLab From fcbcb8a37deec28d938ef5f6122e4022a7365d8e Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Fri, 7 Feb 2025 09:26:18 +0100 Subject: [PATCH 15/19] remove option --- Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py | 1 - Hlt/Moore/python/Moore/LbExec.py | 1 - 2 files changed, 2 deletions(-) diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py index 9a8573cb168..f7076a394fe 100644 --- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py +++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py @@ -322,7 +322,6 @@ def spruce_overlap(options: Options): stream = "default" input_line_config = options.input_streams_attributes_file - # stream_config = options.output_streams_config stream_config = turbolinedict # Open and merge the line_config and the streaming_config diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 03948693a5c..1ba57443371 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -24,7 +24,6 @@ from RecoConf.reconstruction_objects import reconstruction class Options(DefaultOptions): tck: Optional[int] = None - output_streams_config: Optional[str] = "test_streaming_config.json" @contextmanager def apply_binds(self): -- GitLab From b0a274de0115b104a1075085efcb01df874cca88 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Fri, 7 Feb 2025 09:40:01 +0100 Subject: [PATCH 16/19] options corections --- Hlt/Moore/python/Moore/LbExec.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 1ba57443371..06ba9e890cb 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -13,7 +13,7 @@ from enum import Enum from typing import Optional from GaudiConf.LbExec import Options as DefaultOptions -from GaudiConf.LbExec import ProcessTypes, TestOptionsBase +from GaudiConf.LbExec import TestOptionsBase from PyConf.application import ROOT_KEY from PyConf.packing import persistreco_version from PyConf.reading import reconstruction as reconstruction_reading @@ -21,9 +21,20 @@ from PyConf.reading import tes_root from PyConf.reading import upfront_reconstruction as upfront_reconstruction_reading from RecoConf.reconstruction_objects import reconstruction +class ProcessTypes(str, Enum): + Hlt1 = "Hlt1" + Hlt2 = "Hlt2" + TurboPass = "TurboPass" + TurboSpruce = "TurboSpruce" + Spruce = "Spruce" + ReconstructionOnly = "ReconstructionOnly" + Monitoring = "Monitoring" + UserDefined = "UserDefined" + class Options(DefaultOptions): tck: Optional[int] = None + process: Optional[ProcessTypes] = None @contextmanager def apply_binds(self): -- GitLab From 8c5c7b0696b644c37ee10e14e97e777e29ba55d7 Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Fri, 7 Feb 2025 08:40:49 +0000 Subject: [PATCH 17/19] pre-commit fixes patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/50590759 --- Hlt/Moore/python/Moore/LbExec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Hlt/Moore/python/Moore/LbExec.py b/Hlt/Moore/python/Moore/LbExec.py index 06ba9e890cb..0f252926bfd 100644 --- a/Hlt/Moore/python/Moore/LbExec.py +++ b/Hlt/Moore/python/Moore/LbExec.py @@ -21,6 +21,7 @@ from PyConf.reading import tes_root from PyConf.reading import upfront_reconstruction as upfront_reconstruction_reading from RecoConf.reconstruction_objects import reconstruction + class ProcessTypes(str, Enum): Hlt1 = "Hlt1" Hlt2 = "Hlt2" -- GitLab From 37a38bab8c83a73849e908592d74ba81595b8bae Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Tue, 11 Feb 2025 11:15:52 +0100 Subject: [PATCH 18/19] rm tests ffrom CMake --- Hlt/Hlt2Conf/CMakeLists.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Hlt/Hlt2Conf/CMakeLists.txt b/Hlt/Hlt2Conf/CMakeLists.txt index f93c9c94358..e3a8d71bcd0 100644 --- a/Hlt/Hlt2Conf/CMakeLists.txt +++ b/Hlt/Hlt2Conf/CMakeLists.txt @@ -110,21 +110,9 @@ if(BUILD_TESTING AND NOT USE_DD4HEP) Hlt2Conf.sprucing.test_turcal_spruce_2022_data Hlt2Conf.sprucing.test_turcal_spruce_2022_data_pid_check Hlt2Conf.sprucing.test_turcal_spruce_2022_data_pid_rb_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_trackeff_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_trackeff_rb_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_monitoring_check - Hlt2Conf.sprucing.test_turcal_spruce_2022_data_monitoring_rb_check Hlt2Conf.sprucing.test_excl_spruce_2022_data_b2oc_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_bandq_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_qee_check Hlt2Conf.sprucing.test_excl_spruce_2022_data_rd_check - Hlt2Conf.sprucing.test_excl_spruce_2022_data_sl_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_b2cc_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_b2oc_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_bandq_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_bnoc_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_charm_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_qee_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_rd_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_sl_check Hlt2Conf.sprucing.test_excl_spruce_2023_1_data -- GitLab From b0071217bfc52fc63dcbad5d1a74f9c28d8b6427 Mon Sep 17 00:00:00 2001 From: gitlabCI Date: Tue, 11 Feb 2025 11:17:23 +0100 Subject: [PATCH 19/19] rm tests ffrom CMake --- Hlt/Hlt2Conf/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Hlt/Hlt2Conf/CMakeLists.txt b/Hlt/Hlt2Conf/CMakeLists.txt index e3a8d71bcd0..a60aea47f99 100644 --- a/Hlt/Hlt2Conf/CMakeLists.txt +++ b/Hlt/Hlt2Conf/CMakeLists.txt @@ -114,7 +114,6 @@ if(BUILD_TESTING AND NOT USE_DD4HEP) Hlt2Conf.sprucing.test_excl_spruce_2022_data_rd_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_b2oc_check Hlt2Conf.sprucing.test_pass_spruce_2022_data_rd_check - Hlt2Conf.sprucing.test_pass_spruce_2022_data_sl_check Hlt2Conf.sprucing.test_excl_spruce_2023_1_data Hlt2Conf.sprucing.test_excl_spruce_2023_2_data Hlt2Conf.sprucing.test_hlt2_foroverlapcheck -- GitLab