From fa5ea487146d1e67c6e6469cafd2c8259f17f2b0 Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Tue, 12 May 2020 21:18:25 +0200 Subject: [PATCH 1/8] first prototype (not yet working) --- Hlt/Moore/python/Moore/config.py | 4 ++-- Hlt/Moore/python/Moore/selreports.py | 32 ++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index a45613a638f..39b1d16967b 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -221,8 +221,8 @@ def report_writers_node(lines, hlt2 = all(l.name.startswith('Hlt2') for l in lines) assert hlt1 ^ hlt2, 'Expected exclusively all Hlt1 or all Hlt2 lines' if hlt1: - srm = make_selreports(physics_lines, erw) - algs.append(srm) + srm_node, srm = make_selreports(physics_lines, erw) + algs.append(srm_node) srw = HltSelReportsWriter( DecReports=erw.DecReportsLocation, SelReports=srm.SelReports, diff --git a/Hlt/Moore/python/Moore/selreports.py b/Hlt/Moore/python/Moore/selreports.py index c9df0d47cb8..b9aa1020625 100644 --- a/Hlt/Moore/python/Moore/selreports.py +++ b/Hlt/Moore/python/Moore/selreports.py @@ -49,6 +49,11 @@ from PyConf.Algorithms import ( SelReportsMaker, ) +from PyConf.control_flow import ( + CompositeNode, + NodeLogic, +) + __all__ = ["make_selreports"] log = logging.getLogger(__name__) @@ -195,7 +200,8 @@ def line_outputs_for_selreports(line): def make_selreports(lines, decreports, **kwargs): - """Return a SelReportsMaker instance configured for the given lines. + """Return a tuple of a smart SelReportsMaker node plus + a SelReportsMaker instance configured for the given lines. Args: lines (list of HltLine): The lines to create SelReport objects @@ -217,9 +223,31 @@ def make_selreports(lines, decreports, **kwargs): assert len(outputs) == len( set(outputs)), "multiple lines with identical output" - return SelReportsMaker( + srm = SelReportsMaker( DecReports=decreports, SelectionNames=names, CandidateTypes=types, Inputs=outputs, **kwargs) + + def wrap_line(line): + outputs = [ output for output in line_outputs_for_selreports(line) if output ] + if outputs: + return CompositeNode( + 'wrapped_' + line.name, + combineLogic=NodeLogic.LAZY_AND, + children= [line.node] + outputs, + forceOrder=True) + + wrapped_lines = [ wrap_line(line) for line in lines if wrap_line(line) ] + + srm_node = CompositeNode( + 'selreports_maker', + combineLogic=NodeLogic.NONLAZY_AND, + children= wrapped_lines + [srm], + forceOrder=True + ) + + return srm_node, srm + + -- GitLab From 31e5bf552603e6b8010ba2a8746c3958a3f3f90f Mon Sep 17 00:00:00 2001 From: Gitlab CI Date: Tue, 12 May 2020 19:19:05 +0000 Subject: [PATCH 2/8] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/8348934 --- Hlt/Moore/python/Moore/selreports.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Hlt/Moore/python/Moore/selreports.py b/Hlt/Moore/python/Moore/selreports.py index b9aa1020625..06d78f46a2a 100644 --- a/Hlt/Moore/python/Moore/selreports.py +++ b/Hlt/Moore/python/Moore/selreports.py @@ -50,7 +50,7 @@ from PyConf.Algorithms import ( ) from PyConf.control_flow import ( - CompositeNode, + CompositeNode, NodeLogic, ) @@ -200,7 +200,7 @@ def line_outputs_for_selreports(line): def make_selreports(lines, decreports, **kwargs): - """Return a tuple of a smart SelReportsMaker node plus + """Return a tuple of a smart SelReportsMaker node plus a SelReportsMaker instance configured for the given lines. Args: @@ -231,23 +231,22 @@ def make_selreports(lines, decreports, **kwargs): **kwargs) def wrap_line(line): - outputs = [ output for output in line_outputs_for_selreports(line) if output ] - if outputs: + outputs = [ + output for output in line_outputs_for_selreports(line) if output + ] + if outputs: return CompositeNode( - 'wrapped_' + line.name, - combineLogic=NodeLogic.LAZY_AND, - children= [line.node] + outputs, - forceOrder=True) + 'wrapped_' + line.name, + combineLogic=NodeLogic.LAZY_AND, + children=[line.node] + outputs, + forceOrder=True) - wrapped_lines = [ wrap_line(line) for line in lines if wrap_line(line) ] + wrapped_lines = [wrap_line(line) for line in lines if wrap_line(line)] srm_node = CompositeNode( 'selreports_maker', combineLogic=NodeLogic.NONLAZY_AND, - children= wrapped_lines + [srm], - forceOrder=True - ) + children=wrapped_lines + [srm], + forceOrder=True) return srm_node, srm - - -- GitLab From 916d1347c68e433dc6c3e8935ba78515461bdfc1 Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Wed, 27 May 2020 10:19:58 +0200 Subject: [PATCH 3/8] only one line, make structure a bit clenaer --- Hlt/Hlt1Conf/options/hlt1_example.py | 7 +++-- Hlt/Moore/python/Moore/selreports.py | 42 ++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/Hlt/Hlt1Conf/options/hlt1_example.py b/Hlt/Hlt1Conf/options/hlt1_example.py index d1af2554886..2bd92eee661 100644 --- a/Hlt/Hlt1Conf/options/hlt1_example.py +++ b/Hlt/Hlt1Conf/options/hlt1_example.py @@ -20,8 +20,11 @@ options.set_conds_from_testfiledb('MiniBrunel_2018_MinBias_FTv4_DIGI') def make_lines(): - return all_lines() + [ - debug_two_track_mva_line(), +# return [one_track_muon_highpt_line()] + + #return all_lines() + [ + return [ + # debug_two_track_mva_line(), one_track_muon_highpt_line(), ] diff --git a/Hlt/Moore/python/Moore/selreports.py b/Hlt/Moore/python/Moore/selreports.py index 06d78f46a2a..6363cbf6a83 100644 --- a/Hlt/Moore/python/Moore/selreports.py +++ b/Hlt/Moore/python/Moore/selreports.py @@ -54,6 +54,9 @@ from PyConf.control_flow import ( NodeLogic, ) +from PyConf.application import plot_data_flow, plot_control_flow + + __all__ = ["make_selreports"] log = logging.getLogger(__name__) @@ -209,6 +212,11 @@ def make_selreports(lines, decreports, **kwargs): decreports: DecReports data used as input to the SelReports maker. kwargs: Passed to the SelReportsMaker. """ + + print('Called make_selreports') + + + # Gather all outputs along with their type and decision names info = [(line.decision_name, output, output.type) for line in lines for output in line_outputs_for_selreports(line)] @@ -230,23 +238,33 @@ def make_selreports(lines, decreports, **kwargs): Inputs=outputs, **kwargs) - def wrap_line(line): - outputs = [ - output for output in line_outputs_for_selreports(line) if output - ] - if outputs: - return CompositeNode( - 'wrapped_' + line.name, - combineLogic=NodeLogic.LAZY_AND, - children=[line.node] + outputs, - forceOrder=True) + def wrap(line): + print('called wrap_line') + return CompositeNode( + 'dummy_LAZY_AND_' + line.name, + combineLogic=NodeLogic.LAZY_AND, + children=[ line.node] + line_outputs_for_selreports(line), + forceOrder=True) + + def has_output(line): + return line_outputs_for_selreports(line) - wrapped_lines = [wrap_line(line) for line in lines if wrap_line(line)] + #wrapped_lines = [wrap_line(line) for line in lines if wrap_line(line)] + #wrapped_lines = [wrap(line) for line in lines if has_output(line) ] + #print ('AAAAAAAAAAAAAAAAAAAA' , lines[1]) + from Hlt1Conf.lines.high_pt_muon import one_track_muon_highpt_line + # new_line = one_track_muon_highpt_line(name='COPY_Hlt1SingleHighPtMuonLine') + wrapped_lines = [ wrap(line) for line in lines ] srm_node = CompositeNode( 'selreports_maker', combineLogic=NodeLogic.NONLAZY_AND, - children=wrapped_lines + [srm], + #children=wrapped_lines + [srm], + children= wrapped_lines + [srm], forceOrder=True) + + plot_control_flow(srm_node, "TESTCONTROL_SRM") + plot_data_flow([srm_node], "TESTDATA_SRM") + return srm_node, srm -- GitLab From 78ed97018295fad44480f315164779e1042f32d4 Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Wed, 27 May 2020 10:38:24 +0200 Subject: [PATCH 4/8] adding SelReportMaker to BarrierAlgNames --- Hlt/Moore/python/Moore/config.py | 2 +- PyConf/python/PyConf/application.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 39b1d16967b..bed7617ce92 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -322,7 +322,7 @@ def run_moore(options, make_lines, public_tools=[]): config = configure_input(options) lines = make_lines() top_cf_node = moore_control_flow(options, lines) - config.update(configure(options, top_cf_node, public_tools=public_tools)) + config.update(configure(options, top_cf_node, public_tools=public_tools, barrier_algs=['SelReportsMaker'] )) # TODO pass config to gaudi explicitly when that is supported return config diff --git a/PyConf/python/PyConf/application.py b/PyConf/python/PyConf/application.py index cfbb93cb40c..92cfbd623e2 100644 --- a/PyConf/python/PyConf/application.py +++ b/PyConf/python/PyConf/application.py @@ -470,7 +470,7 @@ def assert_empty_dataondemand_service(): assert DataOnDemandSvc().NodeMap == {}, DataOnDemandSvc().NodeMap -def configure(options, control_flow_node, public_tools=[]): +def configure(options, control_flow_node, public_tools=[], barrier_algs=[]): # TODO get rid of magic initial time (must go to configure_input) INITIAL_TIME = 1433509200 @@ -522,7 +522,8 @@ def configure(options, control_flow_node, public_tools=[]): CompositeCFNodes=[node.represent() for node in nodes], MemoryPoolSize=options.memory_pool_size, ThreadPoolSize=options.n_threads, - EnableLegacyMode=options.scheduler_legacy_mode)) + EnableLegacyMode=options.scheduler_legacy_mode, + BarrierAlgNames=barrier_algs)) appMgr = config.add( ApplicationMgr(OutputLevel=options.output_level, EventLoop=scheduler)) appMgr.ExtSvc.insert( -- GitLab From 175b60dfa43ad1d5cdb3a061a153995cf1f01a65 Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Wed, 27 May 2020 12:23:12 +0200 Subject: [PATCH 5/8] remove order --- Hlt/Moore/python/Moore/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index bed7617ce92..8ac8199bdfc 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -302,7 +302,7 @@ def moore_control_flow(options, lines): 'moore', combineLogic=NodeLogic.LAZY_AND, children=[dec] + writers, - forceOrder=True) + forceOrder=False) def run_moore(options, make_lines, public_tools=[]): -- GitLab From 947f68b1ddf26e9f7c39b3aa6e0f5d52fc9d7e6d Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Wed, 27 May 2020 17:11:01 +0200 Subject: [PATCH 6/8] DEBUG --- PyConf/python/PyConf/application.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PyConf/python/PyConf/application.py b/PyConf/python/PyConf/application.py index 92cfbd623e2..9be03eab453 100644 --- a/PyConf/python/PyConf/application.py +++ b/PyConf/python/PyConf/application.py @@ -18,7 +18,7 @@ from six import string_types import pydot import six -from Gaudi.Configuration import appendPostConfigAction, ConfigurableUser, INFO +from Gaudi.Configuration import appendPostConfigAction, ConfigurableUser, INFO, DEBUG import GaudiKernel.ProcessJobOptions from GaudiConf import IOHelper from RawEventFormat import Raw_location_db @@ -523,7 +523,8 @@ def configure(options, control_flow_node, public_tools=[], barrier_algs=[]): MemoryPoolSize=options.memory_pool_size, ThreadPoolSize=options.n_threads, EnableLegacyMode=options.scheduler_legacy_mode, - BarrierAlgNames=barrier_algs)) + BarrierAlgNames=barrier_algs, + OutputLevel=DEBUG)) appMgr = config.add( ApplicationMgr(OutputLevel=options.output_level, EventLoop=scheduler)) appMgr.ExtSvc.insert( -- GitLab From b3848416003c8141fb55d4aa0da725e5453dfc3d Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Wed, 27 May 2020 23:10:27 +0200 Subject: [PATCH 7/8] restore order. only ouput producers in selreporter node --- Hlt/Moore/python/Moore/config.py | 2 +- Hlt/Moore/python/Moore/selreports.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py index 8ac8199bdfc..bed7617ce92 100644 --- a/Hlt/Moore/python/Moore/config.py +++ b/Hlt/Moore/python/Moore/config.py @@ -302,7 +302,7 @@ def moore_control_flow(options, lines): 'moore', combineLogic=NodeLogic.LAZY_AND, children=[dec] + writers, - forceOrder=False) + forceOrder=True) def run_moore(options, make_lines, public_tools=[]): diff --git a/Hlt/Moore/python/Moore/selreports.py b/Hlt/Moore/python/Moore/selreports.py index 6363cbf6a83..b20dc7dd7de 100644 --- a/Hlt/Moore/python/Moore/selreports.py +++ b/Hlt/Moore/python/Moore/selreports.py @@ -240,20 +240,25 @@ def make_selreports(lines, decreports, **kwargs): def wrap(line): print('called wrap_line') + return CompositeNode( 'dummy_LAZY_AND_' + line.name, combineLogic=NodeLogic.LAZY_AND, - children=[ line.node] + line_outputs_for_selreports(line), + #children=[ line.node.children[-1]] + line_outputs_for_selreports(line), + children=[ line.output_producer ] + line_outputs_for_selreports(line), forceOrder=True) def has_output(line): return line_outputs_for_selreports(line) + + #from Hlt1Conf.lines.high_pt_muon import one_track_muon_highpt_line_TEST + #new_line = one_track_muon_highpt_line_TEST(name='COPY_Hlt1SingleHighPtMuonLine').node.children[-1] + #wrapped_lines = [ new_line ] + line_outputs_for_selreports(new_line) + #wrapped_lines = [wrap_line(line) for line in lines if wrap_line(line)] #wrapped_lines = [wrap(line) for line in lines if has_output(line) ] #print ('AAAAAAAAAAAAAAAAAAAA' , lines[1]) - from Hlt1Conf.lines.high_pt_muon import one_track_muon_highpt_line - # new_line = one_track_muon_highpt_line(name='COPY_Hlt1SingleHighPtMuonLine') wrapped_lines = [ wrap(line) for line in lines ] srm_node = CompositeNode( -- GitLab From d14681a6a564f3ed003d34856b5d23975695a4b2 Mon Sep 17 00:00:00 2001 From: Steffen Georg Weber Date: Thu, 28 May 2020 09:29:12 +0200 Subject: [PATCH 8/8] use complete node sans prescaler --- Hlt/Moore/python/Moore/selreports.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Hlt/Moore/python/Moore/selreports.py b/Hlt/Moore/python/Moore/selreports.py index b20dc7dd7de..69cee5241ba 100644 --- a/Hlt/Moore/python/Moore/selreports.py +++ b/Hlt/Moore/python/Moore/selreports.py @@ -244,8 +244,7 @@ def make_selreports(lines, decreports, **kwargs): return CompositeNode( 'dummy_LAZY_AND_' + line.name, combineLogic=NodeLogic.LAZY_AND, - #children=[ line.node.children[-1]] + line_outputs_for_selreports(line), - children=[ line.output_producer ] + line_outputs_for_selreports(line), + children=[ child for child in line.node.children if not child.name.endswith('Prescaler') ] + line_outputs_for_selreports(line), forceOrder=True) def has_output(line): -- GitLab