diff --git a/Hlt/Hlt1Conf/options/hlt1_selreports_filtering.py b/Hlt/Hlt1Conf/options/hlt1_selreports_filtering.py index 37ae4d3fce9452e4ebc92488d8aeadc1001da10d..aa4c0c06b97c85028441c13458fc69ed17244293 100644 --- a/Hlt/Hlt1Conf/options/hlt1_selreports_filtering.py +++ b/Hlt/Hlt1Conf/options/hlt1_selreports_filtering.py @@ -21,7 +21,7 @@ from PyConf.Algorithms import ( TOSFilter__v1__Track, TOSFilter__v1__RecVertex, ) -from PyConf.application import default_raw_event +from PyConf.application import default_raw_banks from Moore import options, run_moore from Moore.config import HltLine @@ -44,7 +44,8 @@ from Functors import ALL def make_selreports(): return HltSelReportsDecoder( - RawEventLocations=default_raw_event(["HltSelReports"]), + DecReports=default_raw_banks("HltDecReports"), + RawBanks=default_raw_banks("HltSelReports"), SourceID='Hlt1', ) diff --git a/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py b/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py index 37178a3956ebd695d4a3cd2448f0a049f4f8c434..3afdca0884eef03e78a525c2b2920e0f336396ae 100644 --- a/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py +++ b/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py @@ -24,7 +24,6 @@ import argparse from collections import defaultdict from Configurables import (ApplicationMgr, HistogramPersistencySvc, IODataManager, LHCbApp) -from DAQSys.Decoders import DecoderDB from GaudiConf import IOHelper import GaudiPython from PyConf.application import configured_ann_svc @@ -41,12 +40,31 @@ IODataManager(DisablePFNWarning=True) # Disable warning about histogram saving not being required HistogramPersistencySvc(OutputLevel=5) # Decode Hlt DecReports -app = ApplicationMgr(TopAlg=[ - DecoderDB["HltDecReportsDecoder/Hlt1DecReportsDecoder"].setup(), - DecoderDB["HltSelReportsDecoder/Hlt1SelReportsDecoder"].setup() -]) +from Configurables import LHCb__UnpackRawEvent, HltDecReportsDecoder, HltSelReportsDecoder +unpacker = LHCb__UnpackRawEvent( + "UnpackRawEvent", + OutputLevel=2, + RawBankLocations=[ + "DAQ/RawBanks/HltDecReports", "DAQ/RawBanks/HltSelReports" + ], + BankTypes=["HltDecReports", "HltSelReports"]) + +decDec = HltDecReportsDecoder( + "HltDecReportsDecoder/Hlt1DecReportsDecoder", + SourceID="Hlt1", + RawBanks=unpacker.RawBankLocations[0]) + +selDec = HltSelReportsDecoder( + "HltSelReportsDecoder/Hlt1SelReportsDecoder", + SourceID="Hlt1", + DecReports=unpacker.RawBankLocations[0], + RawBanks=unpacker.RawBankLocations[1]) + +app = ApplicationMgr( + TopAlg=[unpacker, decDec, selDec], + ExtSvc=[configured_ann_svc(name='HltANNSvc')]) + # decoderDB wants TCKANNSvc as name... -app.ExtSvc += [configured_ann_svc(name='TCKANNSvc')] # Set up counters for recording decisions and selreport existence from MDF counts_from_mdf = defaultdict(lambda: defaultdict(int)) @@ -56,13 +74,13 @@ gaudi.run(1) error = False while TES["/Event"]: - decs = TES["/Event/Hlt1/DecReports"] + decs = TES[str(decDec.OutputHltDecReportsLocation)] if not decs: print("DecReports TES location not found") error = True break - sels = TES["/Event/Hlt1/SelReports"] + sels = TES[str(selDec.OutputHltSelReportsLocation)] if not sels: print("SelReports TES location not found") error = True diff --git a/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py b/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py index 2a2638b340b7a6aa7fce99586355ee3938338372..f534b458b1b5847755649f5f7130e90ebba3e2ad 100644 --- a/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py +++ b/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py @@ -22,7 +22,6 @@ import argparse from collections import defaultdict from Configurables import (ApplicationMgr, HistogramPersistencySvc, IODataManager, LHCbApp) -from DAQSys.Decoders import DecoderDB from GaudiConf import IOHelper import GaudiPython from PyConf.application import configured_ann_svc @@ -56,9 +55,20 @@ IODataManager(DisablePFNWarning=True) # Disable warning about histogram saving not being required HistogramPersistencySvc(OutputLevel=5) # Decode Hlt DecReports -# Configure TCKANNSvc (as that is what DecoderDB wants...) -app = ApplicationMgr( - TopAlg=[DecoderDB["HltDecReportsDecoder/Hlt1DecReportsDecoder"].setup()]) +from Configurables import LHCb__UnpackRawEvent, HltDecReportsDecoder +unpacker = LHCb__UnpackRawEvent( + "UnpackRawEvent", + RawBankLocations=["DAQ/RawBanks/HltDecReports"], + BankTypes=["HltDecReports"]) + +decDec = HltDecReportsDecoder( + "HltDecReportsDecoder/Hlt1DecReportsDecoder", + SourceID="Hlt1", + DecoderMapping="TCKANNSvc", + RawBanks=unpacker.RawBankLocations[0]) + +app = ApplicationMgr(TopAlg=[unpacker, decDec]) +# Configure TCKANNSvc app.ExtSvc += [configured_ann_svc(name='TCKANNSvc')] # Set up counters for recording decisions from MDF @@ -72,7 +82,7 @@ gaudi.run(1) error = False while TES["/Event"]: - decs = TES["/Event/Hlt1/DecReports"] + decs = TES[str(decDec.OutputHltDecReportsLocation)] if not decs: print("DecReports TES location not found") error = True diff --git a/Hlt/Hlt1Conf/tests/options/test_allen_sel_rep_info.py b/Hlt/Hlt1Conf/tests/options/test_allen_sel_rep_info.py index 09b53c2952c1db10798787f44862f7f3403a41ae..88f00ecc8bd16d9d91f3dbb18a81c54bbce2b885 100644 --- a/Hlt/Hlt1Conf/tests/options/test_allen_sel_rep_info.py +++ b/Hlt/Hlt1Conf/tests/options/test_allen_sel_rep_info.py @@ -26,7 +26,6 @@ import re from collections import defaultdict from Configurables import (ApplicationMgr, HistogramPersistencySvc, IODataManager, LHCbApp) -from DAQSys.Decoders import DecoderDB from GaudiConf import IOHelper import GaudiPython from PyConf.application import configured_ann_svc @@ -224,11 +223,29 @@ IODataManager(DisablePFNWarning=True) # Disable warning about histogram saving not being required HistogramPersistencySvc(OutputLevel=5) # Decode Hlt DecReports -app = ApplicationMgr(TopAlg=[ - DecoderDB["HltDecReportsDecoder/Hlt1DecReportsDecoder"].setup(), - DecoderDB["HltSelReportsDecoder/Hlt1SelReportsDecoder"].setup() -]) -# Configure TCKANNSvc (as that is what DecoderDB wants...) +from Configurables import LHCb__UnpackRawEvent, HltDecReportsDecoder, HltSelReportsDecoder +unpacker = LHCb__UnpackRawEvent( + "UnpackRawEvent", + RawBankLocations=[ + "DAQ/RawBanks/HltDecReports", "DAQ/RawBanks/HltSelReports" + ], + BankTypes=["HltDecReports", "HltSelReports"]) + +decDec = HltDecReportsDecoder( + "HltDecReportsDecoder/Hlt1DecReportsDecoder", + SourceID="Hlt1", + DecoderMapping="TCKANNSvc", + RawBanks=unpacker.RawBankLocations[0]) + +selDec = HltSelReportsDecoder( + "HltDecReportsDecoder/Hlt1SelReportsDecoder", + SourceID="Hlt1", + DecoderMapping="TCKANNSvc", + RawBanks=unpacker.RawBankLocations[1]) + +app = ApplicationMgr(TopAlg=[unpacker, decDec, selDec]) + +# Configure TCKANNSvc app.ExtSvc += [configured_ann_svc(name='TCKANNSvc')] gaudi = GaudiPython.AppMgr() @@ -247,7 +264,7 @@ print('Applying objectless line tests to lines matching ', objectless_pattern = re.compile(args.objectless_lines_regex) while TES["/Event"]: - sels = TES["/Event/Hlt1/SelReports"] + sels = TES[str(selDec.OutputHltSelReportsLocation)] if not sels: issues["SelReports TES location not found"] += 1 continue diff --git a/Hlt/Hlt1Conf/tests/options/test_decreports.py b/Hlt/Hlt1Conf/tests/options/test_decreports.py index 3a7238c9be64bb42a182c58353384c8117cc2c7c..1af358eaa7e8bcaf7e3f688cfbb6ec3a51d51954 100644 --- a/Hlt/Hlt1Conf/tests/options/test_decreports.py +++ b/Hlt/Hlt1Conf/tests/options/test_decreports.py @@ -21,7 +21,6 @@ import argparse from Configurables import (ApplicationMgr, HistogramPersistencySvc, IODataManager, LHCbApp) -from DAQSys.Decoders import DecoderDB from GaudiConf import IOHelper import GaudiPython from PyConf.application import configured_ann_svc @@ -59,9 +58,20 @@ IODataManager(DisablePFNWarning=True) HistogramPersistencySvc(OutputLevel=5) # Decode Hlt DecReports -appMgr = ApplicationMgr( - TopAlg=[DecoderDB["HltDecReportsDecoder/Hlt1DecReportsDecoder"].setup()]) -# Configure TCKANNSvc (as that is what DecoderDB wants...) +from Configurables import LHCb__UnpackRawEvent, HltDecReportsDecoder +unpacker = LHCb__UnpackRawEvent( + "UnpackRawEvent", + RawBankLocations=["DAQ/RawBanks/HltDecReports"], + BankTypes=["HltDecReports"]) + +decDec = HltDecReportsDecoder( + "HltDecReportsDecoder/Hlt1DecReportsDecoder", + SourceID="Hlt1", + DecoderMapping="TCKANNSvc", + RawBanks=unpacker.RawBankLocations[0]) + +appMgr = ApplicationMgr(TopAlg=[unpacker, decDec]) +# Configure TCKANNSvc appMgr.ExtSvc += [configured_ann_svc(name='TCKANNSvc')] # Get expected lines from the previous job @@ -79,7 +89,7 @@ gaudi.run(1) error = False while TES["/Event"]: - decs = TES["/Event/Hlt1/DecReports"] + decs = TES[str(decDec.OutputHltDecReportsLocation)] if not decs: error = True break diff --git a/Hlt/Hlt2Conf/options/hlt2_check_output.py b/Hlt/Hlt2Conf/options/hlt2_check_output.py index 703bd4b1421514507136111526858062b1b2ebd1..dedae806c18acc27772c2b45b5a8c63c5956eee5 100644 --- a/Hlt/Hlt2Conf/options/hlt2_check_output.py +++ b/Hlt/Hlt2Conf/options/hlt2_check_output.py @@ -48,6 +48,11 @@ IODataManager(DisablePFNWarning=True) manifest = load_manifest(sys.argv[2]) algs = do_unpacking(manifest, process='Hlt2') +decdecoder = None +for alg in algs: + if "HltDecReportsDecoder" in alg.getFullName(): + decdecoder = alg + appmgr = ApplicationMgr(TopAlg=algs) appmgr.ExtSvc += [configured_ann_svc()] @@ -73,7 +78,7 @@ while TES['/Event']: print('Checking next event.') #TES.dump() - decRep = TES[str(algs[1].OutputHltDecReportsLocation)].decReports() + decRep = TES[str(decdecoder.OutputHltDecReportsLocation)].decReports() for name, report in decRep.items(): if report.decision(): diff --git a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py index 0e3eeae9932bf512ce55da858f9a83e2abd95daa..c42439d1d97a4f0228d191649b2fcbb5a683d54a 100644 --- a/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py +++ b/Hlt/Hlt2Conf/tests/options/sprucing/spruce_check.py @@ -28,7 +28,7 @@ from PyConf.packing import default_persisted_locations from GaudiConf.reading import do_unpacking from GaudiConf.reading import load_manifest -from PyConf.application import configured_ann_svc +from PyConf.application import configured_ann_svc, default_raw_event from Hlt2Conf.check_output import check_persistreco @@ -69,7 +69,7 @@ elif process == "Turbo": RECO_ROOT = "/Event/HLT2" loc = "HLT2" dec_to_check = "PassThrough" - +default_raw_event.global_bind(raw_event_format=0.5) algs = do_unpacking( load_manifest(args.t), process=process, diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 141f76c83a746f8ea19b3c3e315c3d21258e1230..4de7bcfe2310c77bc36a517929abf67ff2ef5c73 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -29,12 +29,13 @@ from PyConf.application import ( root_copy_input_writer, make_odin, default_raw_event, + default_raw_banks, root_writer, make_data_with_FetchDataFromFile, ) from PyConf.filecontent_metadata import generate_encoding_dictionary, register_encoding_dictionary -from GaudiConf.reading import unpack_rawevent, mc_unpackers +from GaudiConf.reading import mc_unpackers from PyConf.utilities import ConfigurationError from PyConf.application import all_nodes_and_algs @@ -159,6 +160,7 @@ def report_writers_nodes(streams, "spruce": "SpruceSelectionID", "pass": "SpruceSelectionID" }[process] + dec_key = int( register_encoding_dictionary( major_name, @@ -166,21 +168,34 @@ def report_writers_nodes(streams, [l.decision_name for l in lines])), 16) # TODO unsigned? Stick to hex string? + ##spruce and passthrough jobs will write a Spruce report + erw = ExecutionReportsWriter( + Persist=[line.name for line in lines], + ANNSvcKey=major_name, + TCK=dec_key # TODO unsigned? Stick to hex string? + ) + drw = HltDecReportsWriter( + SourceID=source_id, + InputHltDecReportsLocation=erw.DecReportsLocation, + EncodingKey=dec_key, + ) + + algs.extend([erw, drw]) if process == "hlt1" or process == "hlt2": - erw = ExecutionReportsWriter( - Persist=[line.name for line in lines], - ANNSvcKey=major_name, - TCK=dec_key, - ) - drw = HltDecReportsWriter( - SourceID=source_id, - InputHltDecReportsLocation=erw.DecReportsLocation, - EncodingKey=dec_key, - ) - algs.extend([erw, drw]) new_hlt_banks['HltDecReports'] = drw.OutputRawEvent + else: + new_hlt_banks['SpruceDecReports'] = drw.OutputView + + if process == "pass": + # For pass, just copy the manifest file, nothing else is needed to be done + if not options.input_manifest_file: + raise RuntimeError( + ' pass-through configuration -- must specify an input manifest' + ) + copy(options.input_manifest_file, options.output_manifest_file) - if process == "hlt1": + elif process == "hlt1": + # For hlt1, make sel reports srm = make_selreports(process, physics_lines, erw) srw = HltSelReportsWriter( SourceID=source_id, @@ -190,7 +205,11 @@ def report_writers_nodes(streams, EncodingKey=srm.properties['EncodingKey']) algs.append(srw) new_hlt_banks['HltSelReports'] = srw.RawEvent - elif process == "hlt2": + + else: + # For hlt2 and spruce, persist_line_outputs + event_stream = "/Event/Spruce" if process == "spruce" else "/Event/HLT2" + reco_stream = "/Event/HLT2" (line_output_cf, line_output_locations, packed_data) = persist_line_outputs( physics_lines, @@ -199,50 +218,16 @@ def report_writers_nodes(streams, associate_mc, process.capitalize(), output_manifest_file, + stream=event_stream, + reco_stream=reco_stream, clone_mc=options.simulation and options.input_type == ROOT_KEY) algs.append(line_output_cf) - new_hlt_banks['DstData'] = packed_data.OutputRawEvent extra_locations_to_persist.extend(line_output_locations) - else: - ##spruce and passthrough jobs will write a Spruce report - erw = ExecutionReportsWriter( - Persist=[line.name for line in lines], - ANNSvcKey=major_name, - TCK=dec_key # TODO unsigned? Stick to hex string? - ) - drw = HltDecReportsWriter( - SourceID=source_id, - InputHltDecReportsLocation=erw.DecReportsLocation, - EncodingKey=dec_key, - ) - - algs.extend([erw, drw]) - new_hlt_banks['SpruceDecReports'] = drw.OutputView - - if process == "spruce": - #Only create new DstData locations if exclusive sprucing and not if passthrough - line_output_cf, line_output_locations, packed_data = persist_line_outputs( - physics_lines, - data_type, - erw.DecReportsLocation, - associate_mc, - process.capitalize(), - output_manifest_file, - stream="/Event/Spruce", - reco_stream="/Event/HLT2", - clone_mc=options.simulation and options.input_type == ROOT_KEY) - + if process == "hlt2": + new_hlt_banks['DstData'] = packed_data.OutputRawEvent + elif process == "spruce": new_hlt_banks['DstData'] = packed_data.OutputView - algs.append(line_output_cf) - extra_locations_to_persist.extend(line_output_locations) - - if process == "pass": - if not options.input_manifest_file: - raise RuntimeError( - ' pass-through configuration -- must specify an input manifest' - ) - copy(options.input_manifest_file, options.output_manifest_file) node = CompositeNode( 'report_writers', @@ -321,7 +306,6 @@ def stream_writer(stream, elif output_type == ROOT_KEY: if process == "spruce" or process == "pass": - locations = [dh.location for dh in new_locations] if process == "spruce": # Do not want DstData raw bank from HLT2 in Sprucing output persist_locations = [ @@ -340,8 +324,8 @@ def stream_writer(stream, extra_locations_to_persist += [mc_part, mc_vert] #persist locations have been unpacked by LHCb__UnpackRawEvent - locations += [ - '/Event/DAQ/RawBanks/%s' % (rb) for rb in persist_locations + locations = new_locations + [ + default_raw_banks(rb) for rb in persist_locations ] consolidate_views = CombineRawBankViewsToRawEvent( @@ -477,7 +461,6 @@ def moore_control_flow(options, streams, process, allen_hlt1, analytics=False): postcaler_decreports_monitor ], force_order=True) - # We want to run the monitoring of line decisions on every event to have the proper normalization. # Therefore, add the DecReportsMonitor after the control flow node containing all selection lines # and use NONLAZY_OR as type of the node containing both. To not run the monitoring on every event @@ -520,21 +503,6 @@ def moore_control_flow(options, streams, process, allen_hlt1, analytics=False): | HLT2_REPORT_RAW_BANK_TYPES) #For spruce and passthrough need to unpack raw event to RawBank::Views if process == "spruce" or process == "pass": - bank_types = list(HLT1_REPORT_RAW_BANK_TYPES - | HLT2_REPORT_RAW_BANK_TYPES - | DETECTOR_RAW_BANK_TYPES) - - if process == "spruce": - # sprucing already unpacked the DstData in reco_objects_for_spruce - bank_types_to_unpack = [ - b for b in bank_types if b != 'DstData' - ] - else: - bank_types_to_unpack = bank_types - unpackrawevent = unpack_rawevent( - bank_types=bank_types_to_unpack, configurables=False) - unpack.append(unpackrawevent) - ## Hack to make `extra_locations_to_persist` objects writable in pass through if options.simulation and options.input_type == 'ROOT' and process == "pass": unpack += mc_unpackers(configurables=False) @@ -894,13 +862,14 @@ def RawEventSize_analysis(process, stream, persist_types, new_locations): algs += [raw_event_combiner] else: - locations = [dh.location for dh in new_locations] # Do not want DstData raw bank from HLT2 in Sprucing output persist_types = [x for x in persist_types if x != 'DstData'] + #persist locations have been unpacked by LHCb__UnpackRawEvent - sizes_of = locations + [ - '/Event/DAQ/RawBanks/%s' % (rb) for rb in persist_types + sizes_of = new_locations + [ + default_raw_banks(rb) for rb in persist_types ] + algs += [ CombineRawBankViewsToRawEvent( name="EvtSize_" + stream, diff --git a/Hlt/Moore/python/Moore/lines.py b/Hlt/Moore/python/Moore/lines.py index 1e3ab916ca1fcc8b76c5c0648cd97ffc401b33b1..f1c341c41de5a8f6a46740895889d6075a84b74e 100644 --- a/Hlt/Moore/python/Moore/lines.py +++ b/Hlt/Moore/python/Moore/lines.py @@ -25,7 +25,7 @@ from Moore.selreports import ( convert_output as convert_output_for_selreports, ) -from PyConf.application import default_raw_event +from PyConf.application import default_raw_banks from .persistence.particle_moving import ( CopyParticles, CopyFlavourTags, dvalgorithm_locations, @@ -320,7 +320,7 @@ class Hlt2Line(DecisionLine): filter_source_id, filter_source_id)) line_regex = "|".join(line for line in hlt_filter_code) hlt_dec_reports = HltDecReportsDecoder( - RawEventLocations=default_raw_event(bank_types=["HltDecReports"]), + RawBanks=default_raw_banks("HltDecReports"), SourceID=filter_source_id) return VoidFilter( name=name, diff --git a/Hlt/Moore/python/Moore/persistence/truth_matching.py b/Hlt/Moore/python/Moore/persistence/truth_matching.py index 4b2336edf0d0eb7b3480292aae397b43ff6706e9..37e7c0040cd8dfa854f8fe99f93667d04bd095e4 100644 --- a/Hlt/Moore/python/Moore/persistence/truth_matching.py +++ b/Hlt/Moore/python/Moore/persistence/truth_matching.py @@ -114,6 +114,7 @@ def _find_protoparticles(candidates): neutral = set() charged_brunel = set() neutral_brunel = set() + # Heuristically determine whether a given container of ProtoParticles was # created from the raw event or was taken from the input file. We must # truth-match differently between the two cases. @@ -138,7 +139,13 @@ def _find_protoparticles(candidates): neutral.add(dh) else: charged.add(dh) - return (charged, neutral), (charged_brunel, neutral_brunel) + + def dh_sorted(handles): + return sorted(handles, key=lambda dh: dh.location) + + return (dh_sorted(charged), + dh_sorted(neutral)), (dh_sorted(charged_brunel), + dh_sorted(neutral_brunel)) def _match_charged(charged, charged_brunel, mc_particles): diff --git a/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py b/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py index 7e355093ad915b8fd5b71b3551890059916c8316..70a297fdf66c889137f28e162f34e1478c0b63c5 100644 --- a/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py +++ b/Hlt/RecoConf/python/RecoConf/calorimeter_reconstruction.py @@ -25,35 +25,37 @@ from PyConf.Algorithms import ( from PyConf.Tools import (CaloFutureECorrection, CaloFutureSCorrection, CaloFutureLCorrection) -from PyConf.application import default_raw_event +from PyConf.application import default_raw_banks from DDDB.CheckDD4Hep import UseDD4Hep -def make_ecal_digits(raw): +def make_ecal_digits(raw, rawerror): ecalpath = '/world/DownstreamRegion/Ecal:DetElement-Info-IOV' if UseDD4Hep else '/dd/Structure/LHCb/DownstreamRegion/Ecal' adc_alg = CaloFutureRawToDigits( name='FutureEcalZSup', - RawEventLocation=raw, + RawBanks=raw, + ErrorRawBanks=rawerror, #need a different property wrt make_hcal_digits and output cannot be set #so DetectorLocation has to be set here DetectorLocation=ecalpath) return adc_alg.OutputDigitData -def make_hcal_digits(raw): +def make_hcal_digits(raw, rawerror): hcalpath = '/world/DownstreamRegion/Hcal:DetElement-Info-IOV' if UseDD4Hep else '/dd/Structure/LHCb/DownstreamRegion/Hcal' adc_alg_Hcal = CaloFutureRawToDigits( name='FutureHcalZSup', - RawEventLocation=raw, + RawBanks=raw, + ErrorRawBanks=rawerror, #need a different property wrt make_ecal_digits and output cannot be set #so DetectorLocation has to be set here DetectorLocation=hcalpath) return adc_alg_Hcal.OutputDigitData -def make_digits(raw): - digitsEcal = make_ecal_digits(raw) - digitsHcal = make_hcal_digits(raw) +def make_digits(raw, rawerror): + digitsEcal = make_ecal_digits(raw, rawerror) + digitsHcal = make_hcal_digits(raw, rawerror) return {"digitsEcal": digitsEcal, "digitsHcal": digitsHcal} @@ -442,7 +444,7 @@ def make_merged_pi0_various(ecalClusters, pvs): @configurable def make_calo(tracks, pvs, - make_raw=default_raw_event, + make_raw=default_raw_banks, calo_raw_event=False, chargedpid_types={ "calo": ["Long", "Downstream"], @@ -451,15 +453,19 @@ def make_calo(tracks, trackrels=None): # digits if calo_raw_event: # needed for real data - rawEventEcal = make_raw(["Calo"]) - rawEventHcal = make_raw(["Calo"]) + rawEventCalo = make_raw("Calo") + rawEventCaloError = make_raw("CaloError") + rawToDigitsOutput = make_digits(rawEventCalo, rawEventCaloError) + digitsEcal = rawToDigitsOutput["digitsEcal"] + digitsHcal = rawToDigitsOutput["digitsHcal"] else: - rawEventEcal = make_raw(["EcalPacked"]) - rawEventHcal = make_raw(["HcalPacked"]) - rawToDigitsOutputEcal = make_digits(rawEventEcal) - rawToDigitsOutputHcal = make_digits(rawEventHcal) - digitsEcal = rawToDigitsOutputEcal["digitsEcal"] - digitsHcal = rawToDigitsOutputHcal["digitsHcal"] + rawEventEcal = make_raw("EcalPacked") + rawEventHcal = make_raw("HcalPacked") + rawEventEcalError = make_raw("EcalPackedError") + rawEventHcalError = make_raw("HcalPackedError") + + digitsEcal = make_ecal_digits(rawEventEcal, rawEventEcalError) + digitsHcal = make_hcal_digits(rawEventHcal, rawEventHcalError) # clusters ecalClusters = make_clusters(digitsEcal) @@ -561,13 +567,13 @@ def make_calo(tracks, } -def make_calo_reduced(make_raw=default_raw_event): - rawEventEcal = make_raw(["EcalPacked"]) - rawEventHcal = make_raw(["HcalPacked"]) - rawToDigitsOutputEcal = make_digits(rawEventEcal) - rawToDigitsOutputHcal = make_digits(rawEventHcal) - digitsEcal = rawToDigitsOutputEcal["digitsEcal"] - digitsHcal = rawToDigitsOutputHcal["digitsHcal"] +def make_calo_reduced(make_raw=default_raw_banks): + rawEventEcal = make_raw("EcalPacked") + rawEventHcal = make_raw("HcalPacked") + rawEventEcalError = make_raw("EcalPackedError") + rawEventHcalError = make_raw("HcalPackedError") + digitsEcal = make_ecal_digits(rawEventEcal, rawEventEcalError) + digitsHcal = make_hcal_digits(rawEventHcal, rawEventHcalError) clusters = make_clusters(digitsEcal) return { @@ -577,10 +583,11 @@ def make_calo_reduced(make_raw=default_raw_event): } -def make_calo_resolution_gamma(tracks, pvs, make_raw=default_raw_event): - rawEventEcal = make_raw(["EcalPacked"]) - rawToDigitsOutputEcal = make_digits(rawEventEcal) - digitsEcal = rawToDigitsOutputEcal["digitsEcal"] +def make_calo_resolution_gamma(tracks, pvs, make_raw=default_raw_banks): + rawEventEcal = make_raw("EcalPacked") + rawEventEcalError = make_raw("EcalPackedError") + digitsEcal = make_ecal_digits(rawEventEcal, rawEventEcalError) + clusters = make_clusters_various(digitsEcal) # track-cluster matching input @@ -612,9 +619,11 @@ def make_calo_resolution_gamma(tracks, pvs, make_raw=default_raw_event): } -def make_calo_cluster_shapes(tracks, pvs, make_raw=default_raw_event): - rawEventEcal = make_raw(["EcalPacked"]) - digitsEcal = make_digits(rawEventEcal)["digitsEcal"] +def make_calo_cluster_shapes(tracks, pvs, make_raw=default_raw_banks): + rawEventEcal = make_raw("EcalPacked") + rawEventEcalError = make_raw("EcalPackedError") + + digitsEcal = make_ecal_digits(rawEventEcal, rawEventEcalError) clusters = make_clusters_shapes(digitsEcal) tracks_incalo = make_acceptance(tracks) @@ -633,10 +642,10 @@ def make_calo_cluster_shapes(tracks, pvs, make_raw=default_raw_event): } -def make_calo_resolution_pi0(pvs, make_raw=default_raw_event): - rawEventEcal = make_raw(["EcalPacked"]) - rawToDigitsOutputEcal = make_digits(rawEventEcal) - digitsEcal = rawToDigitsOutputEcal["digitsEcal"] +def make_calo_resolution_pi0(pvs, make_raw=default_raw_banks): + rawEventEcal = make_raw("EcalPacked") + rawEventEcalError = make_raw("EcalPackedError") + digitsEcal = make_ecal_digits(rawEventEcal, rawEventEcalError) clusters = make_clusters_various(digitsEcal) # default Cellular Automaton diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py index b633a18c80d9abf560b1109cacc66e556936d5a1..f3650c57ed4fe18d95b79bcb072767413f26dbd5 100644 --- a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py +++ b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py @@ -12,6 +12,7 @@ import importlib from PyConf import configurable from PyConf.components import Algorithm from PyConf.control_flow import CompositeNode +from PyConf.application import default_raw_banks from PyConf.Algorithms import ( LHCb__Converters__Track__v1__fromVectorLHCbTrack as TrkConvVecV1ToKeyContV1, @@ -124,15 +125,20 @@ def call_allen_decision_logger(): def decode_allen_dec_reports(raw_dec_reports=make_allen_raw_reports): from PyConf.Algorithms import HltDecReportsDecoder + decoder = HltDecReportsDecoder( - RawEventLocations=raw_dec_reports(), SourceID='Hlt1') + RawBanks=default_raw_banks("HltDecReports", raw_dec_reports), + SourceID='Hlt1') return decoder def decode_allen_sel_reports(raw_sel_reports=make_allen_raw_reports): from PyConf.Algorithms import HltSelReportsDecoder + decoder = HltSelReportsDecoder( - RawEventLocations=raw_sel_reports(), SourceID='Hlt1') + RawBanks=default_raw_banks("HltSelReports", raw_sel_reports), + DecReports=default_raw_banks("HltDecReports", raw_sel_reports), + SourceID='Hlt1') return decoder diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_muonid.py b/Hlt/RecoConf/python/RecoConf/hlt1_muonid.py index 41638ea834f212ecdffa94961ef01e25b61580cc..1b06cddc4718b93d7ae6945b7623e2dc02c1faad 100644 --- a/Hlt/RecoConf/python/RecoConf/hlt1_muonid.py +++ b/Hlt/RecoConf/python/RecoConf/hlt1_muonid.py @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from PyConf import configurable -from PyConf.application import default_raw_event +from PyConf.application import default_raw_banks, default_raw_event from PyConf.Algorithms import ( MuonRawToHits, MuonRawInUpgradeToHits, @@ -27,7 +27,8 @@ def make_muon_hits(geometry_version=2, make_raw=default_raw_event): else: raise ValueError("Unsupported muon decoding version") - return raw_to_hits(RawEventLocation=make_raw(["Muon"])).HitContainer + return raw_to_hits( + RawBanks=default_raw_banks("Muon", make_raw)).HitContainer @configurable diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py b/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py index 9316f1fb476a2617c218b7433a9c24f1ecb90072..852a220f0b5a5b391ca257c2e50e7e8626aeb061 100644 --- a/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py +++ b/Hlt/RecoConf/python/RecoConf/hlt1_tracking.py @@ -76,7 +76,6 @@ def require_gec(make_raw=default_raw_banks, cut=-1, **kwargs): @configurable def make_VeloClusterTrackingSIMD(algorithm=VeloClusterTrackingSIMD, raw_event=default_raw_event, - detector=["VP"], masked_sensors=[]): """Simple helper to make sure both, make_VeloClusterTrackingSIMD_tracks and make_VeloClusterTrackingSIMD_hits, access the identically configured version of VeloClusterTrackingSIMD @@ -90,10 +89,16 @@ def make_VeloClusterTrackingSIMD(algorithm=VeloClusterTrackingSIMD, The Velo tracking algorithm. """ + # For VeloRetinaClusterTrackingSIMD use VPRetinaClusters rawbank in the file + if "Retina" in str(algorithm): + bank_type = "VPRetinaCluster" + # In case raw_event is made from VP or algorithm is VeloClusterTrackingSIMD, use VP rawbank in the event + else: + bank_type = "VP" my_SensorMasks = [j in masked_sensors for j in range(208) ] # 208 = LHCb::Pr::Velo::VPInfos::NSensors return algorithm( - RawEventLocation=raw_event(detector), + RawBanks=default_raw_banks(bank_type, raw_event), SensorMasks=tuple(my_SensorMasks)) @@ -146,8 +151,9 @@ def make_SPmixed_raw_event(detector=["VP"], make_raw=default_raw_event): Returns: """ + # Can detector ever be something else return VPRetinaSPmixer( - RawEventLocation=make_raw(detector)).RawEventLocationMixed + RawEventLocation=make_raw(["VP"])).RawEventLocationMixed @configurable diff --git a/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py b/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py index 70a2ee0bcc43f2aebf8bb9b4823a0e63a27a201d..f69a19fcbbb7a1dbb8645196bfef2dde7d528b76 100644 --- a/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py +++ b/Hlt/RecoConf/python/RecoConf/reco_objects_for_spruce.py @@ -44,13 +44,14 @@ def upfront_reconstruction(simulation=True): unpackers = reading.unpackers( reading.make_locations(m, stream), m, - upfront_decoder(process="Hlt2"), + upfront_decoder(process="Hlt2", stream=None), configurables=False) ### TODO:FIXME take advantage of the fact that the above have datahandles... # i.e. should _not_ have to return decoder here, and should just return the _output handles_ and not the algorithms # i.e. `upfront_reconstruction` should be a drop-in replacement for `reconstruction()`, with the same return type - return [upfront_decoder(process="Hlt2").producer] + mc_algs + unpackers + return [upfront_decoder(process="Hlt2", stream=None).producer + ] + mc_algs + unpackers def reconstruction(simulation=True): diff --git a/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py b/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py index cc659560e43f437b66c233b4debe59c260d1b624..d11eaedb85a24925c52f7057bdc37c9de59b64f7 100644 --- a/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py +++ b/Hlt/RecoConf/python/RecoConf/rich_reconstruction.py @@ -10,7 +10,7 @@ ############################################################################### from PyConf import configurable from PyConf.packing import persisted_location -from PyConf.application import default_raw_event +from PyConf.application import default_raw_banks from Configurables import Rich__Future__ParticleProperties as PartProps from PyConf.Algorithms import ( @@ -230,13 +230,13 @@ def shortTrackType(track_name): @configurable -def make_rich_pixels(options, make_raw=default_raw_event): +def make_rich_pixels(options, make_raw=default_raw_banks): """ Return pixel specific RICH data. Args: options (dict): The processing options to use - make_raw : The LHCb RawEvent data object + make_raw : The LHCb RawBanks data object Returns: dict of useful data locations. @@ -248,19 +248,19 @@ def make_rich_pixels(options, make_raw=default_raw_event): # The conf dict to return results = {} - # Get the raw event - rawEvent = make_raw(["Rich"]) + # Get the raw banks + rawBanks = make_raw("Rich") # Get the detector options det_opts = get_detector_bool_opts(options) # document raw event and Odin used - results["RawEvent"] = rawEvent + results["RawBanks"] = rawBanks # Decode the Raw event to RichSmartIDs richDecode = RichDecoder( name="RichRawDecoder", - RawEventLocation=rawEvent, + RawBanks=rawBanks, Detectors=det_opts, Panels=options["ActivatePanel"]) results["RichDecodedData"] = richDecode.DecodedDataLocation @@ -467,7 +467,7 @@ def make_rich_tracks(track_name, input_tracks, options): def make_rich_photons(track_name, input_tracks, options, - make_raw=default_raw_event): + make_raw=default_raw_banks): """ Return reconstructed photon specific RICH data. @@ -475,7 +475,7 @@ def make_rich_photons(track_name, track_name (str): The name to assign to this configuration input_tracks (dict): The input tracks to process options (dict): The processing options to use - make_raw : The entity that provides the RawEvent to use (??) + make_raw : The entity that provides the RawBanks to use (??) Returns: dict of useful data locations. @@ -552,7 +552,7 @@ def make_rich_photons(track_name, def make_rich_pids(track_name, input_tracks, options, - make_raw=default_raw_event): + make_raw=default_raw_banks): """ Return RICH PID data. @@ -560,7 +560,7 @@ def make_rich_pids(track_name, track_name (str): The name to assign to this configuration input_tracks (dict): The input tracks to process options (dict): The processing options to use - make_raw : The entity that provides the RawEvent to use (??) + make_raw : The entity that provides the RawBanks to use (??) Returns: dict of useful data locations. diff --git a/Hlt/RecoConf/python/RecoConf/standalone.py b/Hlt/RecoConf/python/RecoConf/standalone.py index f9cd919423649120dfe30eb6ddee8acf8bf015e3..6d5ca2aca84ef1099a4b96326fc2bf3cf05d2829 100644 --- a/Hlt/RecoConf/python/RecoConf/standalone.py +++ b/Hlt/RecoConf/python/RecoConf/standalone.py @@ -40,7 +40,7 @@ from .calorimeter_mc_checking import ( check_calo_efficiency_fromB, check_calo_charged_pids) from .rich_reconstruction import make_rich_pixels, default_rich_reco_options -from PyConf.application import default_raw_event, make_odin, make_data_with_FetchDataFromFile +from PyConf.application import default_raw_event, default_raw_banks, make_odin, make_data_with_FetchDataFromFile from PyConf.packing import persisted_location from PyConf.Algorithms import ( VeloRetinaClusterTrackingSIMD, @@ -133,9 +133,11 @@ def phoenix_data_dump(): VP_hits = make_VeloClusterTrackingSIMD_hits() FT_hits = make_PrStoreSciFiHits_hits() odin = make_odin() - raw = default_raw_event(["VP"]) - calo_Edig = make_ecal_digits(raw) - calo_Hdig = make_hcal_digits(raw) + + calo_Edig = make_ecal_digits( + default_raw_banks("EcalPacked"), default_raw_banks("EcalPackedError")) + calo_Hdig = make_hcal_digits( + default_raw_banks("HcalPacked"), default_raw_banks("HcalPackedError")) ### Algorithms in Rec/RecAlgs Project to dump the data into Phoenix .json format @@ -190,9 +192,11 @@ def phoenix_data_dump_hlt2(): FT_hits = make_PrStoreSciFiHits_hits() Muon_hits = make_muon_hits() odin = make_odin() - raw = default_raw_event(["VP"]) - calo_Edig = make_ecal_digits(raw) - calo_Hdig = make_hcal_digits(raw) + + calo_Edig = make_ecal_digits( + default_raw_banks("EcalPacked"), default_raw_banks("EcalPackedError")) + calo_Hdig = make_hcal_digits( + default_raw_banks("HcalPacked"), default_raw_banks("HcalPackedError")) rich_data = make_rich_pixels(default_rich_reco_options()) # Add proto particles