From fa73e141ae0347395f51d80370bb5006b1108cf9 Mon Sep 17 00:00:00 2001 From: Andreas Pappas Date: Wed, 28 Jul 2021 15:08:08 +0200 Subject: [PATCH 1/4] create phoenix_data_dump method to appropriate make use of the cplusplus algs to output raw LHCb event data to json format see the other 2 MRs in Rec !2537 and LHCb !3241 add dumpEventToJson python class make appropriate changes to include the new Phoenix Sink Algorithm Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/15817253 remove TestDumper from includes change default dataset filename in Phoenix add all the new Algorithms to be compiled :white_check_mark: Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/16168643 remove extra options, stick to defaults add qmt test for the phoenix_data_dump option add phoenix data_dump option with mc_checking flag modify phoenix event data dump to be enabled with mc_checking flag note: must be run with it's own phoenix_event_data_dump.py option file, so the mc_checking flag can be enabled and thus the data will be dumped. Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/16459223 remove mc_checking flag Note: It is not needed since we got a separate phoenix_data_dump method with it's own option file fix phoenix_data_dump method use set_evt_max_to_5 to set the number of events to be run in the options file Change suggested from Mr. Hasse to be used instead of modifying the download_mdf_input file add a default value to the raw_event method I was getting an error about an empty bank if I were not to provide a default. See the following from the standalone.py file line 342 TODO below we rely on the fact that the RawEvent containing "VP" also has the other banks. The proper fix would be to have the RawBankSizeMonitor accept multiple RawEvents (or even RawBank::View's) RawBankSizeMonitor(RawEventLocation=default_raw_event(["VP"])) Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/17190837 refactor a bit add Calo to the list of the default raw events every event data that is going to be used in Phoenix must be listed in the default_raw_event method apparenlty one needs to provide only 1 item in the default_raw_event --- .../options/phoenix_event_data_dump.py | 17 +++++++ Hlt/RecoConf/python/RecoConf/standalone.py | 46 ++++++++++++++++++- .../tests/qmtest/phoenix_event_data_dump.qmt | 36 +++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 Hlt/RecoConf/options/phoenix_event_data_dump.py create mode 100644 Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt diff --git a/Hlt/RecoConf/options/phoenix_event_data_dump.py b/Hlt/RecoConf/options/phoenix_event_data_dump.py new file mode 100644 index 00000000000..b7bd232f3e2 --- /dev/null +++ b/Hlt/RecoConf/options/phoenix_event_data_dump.py @@ -0,0 +1,17 @@ +############################################################################### +# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +from Moore import options, run_reconstruction +from RecoConf.standalone import phoenix_data_dump + +# will give the filename for use in PhoenixSink (replacing the default one) +options.phoenix_filename = "LHCb_EventDataset.json" + +run_reconstruction(options, phoenix_data_dump) diff --git a/Hlt/RecoConf/python/RecoConf/standalone.py b/Hlt/RecoConf/python/RecoConf/standalone.py index dfed7d57ce2..964878f36ff 100644 --- a/Hlt/RecoConf/python/RecoConf/standalone.py +++ b/Hlt/RecoConf/python/RecoConf/standalone.py @@ -27,13 +27,14 @@ from .hlt2_tracking import ( get_default_tracks_for_calo, get_default_out_track_types_for_light_reco, get_UpgradeGhostId_tool, make_PrForwardTracking_tracks) from .calorimeter_reconstruction import (make_calo, make_calo_resolution_gamma, - make_calo_resolution_pi0) + make_calo_resolution_pi0, + make_ecal_digits, make_hcal_digits) from .calorimeter_mc_checking import ( monitor_calo_cluster_resolution, monitor_calo_future_cluster_resolution, monitor_calo_photon_resolution, monitor_calo_pi0_resolution, check_calo_cluster_efficiency, check_calo_hypo_efficiency) -from PyConf.application import default_raw_event +from PyConf.application import default_raw_event, make_odin from PyConf.Algorithms import ( VeloRetinaClusterTrackingSIMD, CaloFutureDigit2MCLinks2Table, CaloClusterMCTruth, TracksToSelection, TrackSelectionMerger, @@ -56,6 +57,11 @@ from GaudiKernel.SystemOfUnits import MeV, mm from Moore.config import Reconstruction from .data_from_file import boole_links_digits_mcparticles, mc_unpackers from .rich_add_reconstruction_monitoring_checking import add_hlt2_rich +from PyConf.Algorithms import ( + LHCb__Phoenix__DumpPlanarCaloCellsToJson, + LHCb__Phoenix__DumpVPHitEvent, + LHCb__Phoenix__DumpTracksEvent, +) @configurable @@ -95,6 +101,42 @@ def standalone_hlt1_reco(do_mc_checking=False): return Reconstruction('hlt1_reco', data, reco_prefilters()) +@configurable +def phoenix_data_dump(): + """ Runs only components needed to produce run3 event data (eg. run3 VPHits) + Args: + None + Returns: + Reconstruction: Run3 Event Data To be used in LHCb / Phoenix Web Event / Geometry Display + + """ + hlt1_tracks = make_hlt1_tracks() + fitted_tracks = make_VeloKalman_fitted_tracks(hlt1_tracks) + VP_hits = make_VeloClusterTrackingSIMD_hits() + odin = make_odin() + raw = default_raw_event(["VP"]) + calo_Edig = make_ecal_digits(raw) + calo_Hdig = make_hcal_digits(raw) + + ## The following was the old dumper / way of dumping event data to json (before sink introduction) + #test_dumper = LHCb__TestDumper(InputTracksName=fitted_tracks["v2"], VPHitsLocation=VP_hits, CaloEDigits=calo_Edig, CaloHDigits=calo_Hdig, ODIN=odin, TestFileName="testLHCbEventData.json") + + ### Algorithms in Rec/RecAlgs Project to dump the data into Phoenix .json format + + planar_dumper = LHCb__Phoenix__DumpPlanarCaloCellsToJson( + CaloEDigits=calo_Edig, CaloHDigits=calo_Hdig, ODIN=odin) + + vp_dumper = LHCb__Phoenix__DumpVPHitEvent( + VPHitsLocation=VP_hits, ODIN=odin) + + tracks_dumper = LHCb__Phoenix__DumpTracksEvent( + InputTracksName=fitted_tracks["v1"], ODIN=odin) + + data = [planar_dumper, vp_dumper, tracks_dumper] + + return Reconstruction('data_dump', data, reco_prefilters()) + + @configurable def standalone_hlt1_reco_retinacluster(do_mc_checking=False): """ Run the Hlt1 reconstruction, i.e. tracking and muon id, using Retina Cluster, MC checking optional diff --git a/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt new file mode 100644 index 00000000000..46abfef49f1 --- /dev/null +++ b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt @@ -0,0 +1,36 @@ + + + + +gaudirun.py + + $MOOREROOT/options/force_functor_cache.py + $MOOREROOT/tests/options/mdf_input_and_conds.py + $MOOREROOT/tests/options/download_mdf_input.py + + $MOOREROOT/tests/options/set_evt_max_to_5.py + $RECOCONFROOT/options/phoenix_event_data_dump.py + +true +../refs/phoenix_event_data_dump.ref +../refs/empty.ref + + +from Moore.qmtest.exclusions import ref_preprocessor +validateWithReference(preproc = ref_preprocessor) + +from Moore.qmtest.exclusions import remove_known_warnings +countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, + stdout=remove_known_warnings(stdout)) + + + -- GitLab From b6d8dafb78d33618f0c43c7e008cd65545d5eb85 Mon Sep 17 00:00:00 2001 From: Andreas Date: Tue, 9 Nov 2021 14:09:59 +0100 Subject: [PATCH 2/4] refactor --- Hlt/RecoConf/python/RecoConf/standalone.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Hlt/RecoConf/python/RecoConf/standalone.py b/Hlt/RecoConf/python/RecoConf/standalone.py index 964878f36ff..15ac9bfa0d9 100644 --- a/Hlt/RecoConf/python/RecoConf/standalone.py +++ b/Hlt/RecoConf/python/RecoConf/standalone.py @@ -118,9 +118,6 @@ def phoenix_data_dump(): calo_Edig = make_ecal_digits(raw) calo_Hdig = make_hcal_digits(raw) - ## The following was the old dumper / way of dumping event data to json (before sink introduction) - #test_dumper = LHCb__TestDumper(InputTracksName=fitted_tracks["v2"], VPHitsLocation=VP_hits, CaloEDigits=calo_Edig, CaloHDigits=calo_Hdig, ODIN=odin, TestFileName="testLHCbEventData.json") - ### Algorithms in Rec/RecAlgs Project to dump the data into Phoenix .json format planar_dumper = LHCb__Phoenix__DumpPlanarCaloCellsToJson( -- GitLab From c0ba3b99ebf4332f577c2b94f215e8d2b0c955da Mon Sep 17 00:00:00 2001 From: Sebastien Ponce Date: Thu, 9 Dec 2021 08:49:30 +0100 Subject: [PATCH 3/4] Dropped usage of reference file. A validation of the generated json file will come in another MR --- Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt index 46abfef49f1..0607be9bde2 100644 --- a/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt +++ b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt @@ -24,10 +24,6 @@ ../refs/phoenix_event_data_dump.ref ../refs/empty.ref - -from Moore.qmtest.exclusions import ref_preprocessor -validateWithReference(preproc = ref_preprocessor) - from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, stdout=remove_known_warnings(stdout)) -- GitLab From d88f13dba0b6804cb4c07a0061b90a8b179bc589 Mon Sep 17 00:00:00 2001 From: Rosen Matev Date: Thu, 9 Dec 2021 08:56:38 +0100 Subject: [PATCH 4/4] Apply 1 suggestion(s) to 1 file(s) --- Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt | 1 - 1 file changed, 1 deletion(-) diff --git a/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt index 0607be9bde2..4dc4021183e 100644 --- a/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt +++ b/Hlt/RecoConf/tests/qmtest/phoenix_event_data_dump.qmt @@ -21,7 +21,6 @@ $RECOCONFROOT/options/phoenix_event_data_dump.py true -../refs/phoenix_event_data_dump.ref ../refs/empty.ref from Moore.qmtest.exclusions import remove_known_warnings -- GitLab