From f674b781c127b4e4855ce0d608001ec2711ebfc1 Mon Sep 17 00:00:00 2001 From: sesen Date: Fri, 30 Sep 2022 16:41:02 +0200 Subject: [PATCH 1/7] enable checksum tests --- Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py | 7 +++---- .../qmtest/test_hlt2_check_packed_data_checksums.qmt | 9 --------- Hlt/Moore/python/Moore/persistence/packing.py | 4 +++- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py b/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py index a034c8b2903..b24f1d04d25 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py +++ b/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py @@ -12,8 +12,7 @@ """ from Moore import options, run_moore -from PyConf.Algorithms import HltPackedBufferWriter -from Gaudi.Configuration import DEBUG +from Moore.persistence.packing import pack_stream_objects from RecoConf.global_tools import stateProvider_with_simplified_geom from RecoConf.reconstruction_objects import reconstruction @@ -29,6 +28,6 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind(from_file=False), HltPackedBufferWriter.bind( - OutputLevel=DEBUG, EnableChecksum=True): +with reconstruction.bind(from_file=False), pack_stream_objects.bind( + enable_checksum=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt index 17d434bf209..ec0f39ac45e 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt @@ -33,13 +33,4 @@ validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_schedul - - - .* - - - diff --git a/Hlt/Moore/python/Moore/persistence/packing.py b/Hlt/Moore/python/Moore/persistence/packing.py index a8024185d06..569be20223f 100644 --- a/Hlt/Moore/python/Moore/persistence/packing.py +++ b/Hlt/Moore/python/Moore/persistence/packing.py @@ -27,7 +27,8 @@ def pack_stream_objects(stream, inputs, encoding_key, source_id, - enable_check=False): + enable_check=False, + enable_checksum=False): """Return CF node that packs and serialises a set of containers to a raw bank. Args: stream (str): TES root containing objects to be persisted. @@ -46,6 +47,7 @@ def pack_stream_objects(stream, packer = p( InputName=[force_location(loc) for loc in inputs[t]], EnableCheck=enable_check, + EnableChecksum=enable_checksum, EncodingKey=encoding_key) packers += [packer] -- GitLab From 514a8ee5c6b4364fe799e2cc205a8c2ec4000a6f Mon Sep 17 00:00:00 2001 From: sesen Date: Mon, 3 Oct 2022 11:42:25 +0200 Subject: [PATCH 2/7] remove flucctuating counters from trackeff tests --- Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt | 4 ++-- Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt index d6a77f543e1..c74f1bbdb82 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt @@ -23,8 +23,8 @@ Make sure the HLT2 line example configures and runs without errors 1200 -from Moore.qmtest.exclusions import ref_preprocessor -validateWithReference(preproc = ref_preprocessor) +from Moore.qmtest.exclusions import ref_preprocessor, remove_known_fluctuating_counters +validateWithReference(preproc = ref_preprocessor, counter_preproc = remove_known_fluctuating_counters) from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt index 95c8d4513fd..5739843b3b4 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt @@ -23,8 +23,8 @@ Make sure the HLT2 line example configures and runs without errors 1200 -from Moore.qmtest.exclusions import ref_preprocessor -validateWithReference(preproc = ref_preprocessor) +from Moore.qmtest.exclusions import ref_preprocessor, remove_known_fluctuating_counters +validateWithReference(preproc = ref_preprocessor, counter_preproc = remove_known_fluctuating_counters) from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, -- GitLab From 31acf7b5cebdbdc51c09e8d030935f49f04d1ab9 Mon Sep 17 00:00:00 2001 From: sesen Date: Mon, 3 Oct 2022 12:23:27 +0200 Subject: [PATCH 3/7] add serialized data counter to exlcusions, remove the counter exclusion from checksum test --- .../tests/qmtest/test_hlt2_check_packed_data_checksums.qmt | 2 +- Hlt/Moore/python/Moore/qmtest/exclusions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt index ec0f39ac45e..9e0cf24abec 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt @@ -29,7 +29,7 @@ countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, from Moore.qmtest.exclusions import skip_initialize, skip_scheduler, remove_known_fluctuating_counters from RecConf.QMTest.exclusions import preprocessor as RecPreprocessor -validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler, counter_preproc = remove_known_fluctuating_counters) +validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler) diff --git a/Hlt/Moore/python/Moore/qmtest/exclusions.py b/Hlt/Moore/python/Moore/qmtest/exclusions.py index 0890d70e580..48c16221765 100644 --- a/Hlt/Moore/python/Moore/qmtest/exclusions.py +++ b/Hlt/Moore/python/Moore/qmtest/exclusions.py @@ -62,7 +62,8 @@ remove_known_fluctuating_counters = LineSkipper(regexps=[ # the size of the compressed data buffer fluctuates slightly (~0.001%) between runs. # The same does not happen for the uncompressed data buffer. # See https://gitlab.cern.ch/lhcb/LHCb/-/issues/151 - r' \| "Size of compressed data" .*' + r' \| "Size of compressed data" .*', + r' \| "Size of serialized data" .*' ]) # In most cases everything interesting happens at finalize: counters, -- GitLab From 8126b6c37c8d133281737f21dd9a9b470e5725c8 Mon Sep 17 00:00:00 2001 From: sesen Date: Mon, 3 Oct 2022 17:24:23 +0200 Subject: [PATCH 4/7] update the note on fluctuating counters --- Hlt/Moore/python/Moore/qmtest/exclusions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Hlt/Moore/python/Moore/qmtest/exclusions.py b/Hlt/Moore/python/Moore/qmtest/exclusions.py index 48c16221765..4edb606e0ff 100644 --- a/Hlt/Moore/python/Moore/qmtest/exclusions.py +++ b/Hlt/Moore/python/Moore/qmtest/exclusions.py @@ -58,10 +58,8 @@ remove_known_fluctuating_counters = LineSkipper(regexps=[ # reconstruction tests based on HLT2 line content r' \| "# loaded from (?:CACHE|PYTHON)" .*', # The HltPackedDataWriter uses two data buffers: one that holds uncompressed - # data and another that holds this data after compression. For some reason, - # the size of the compressed data buffer fluctuates slightly (~0.001%) between runs. - # The same does not happen for the uncompressed data buffer. - # See https://gitlab.cern.ch/lhcb/LHCb/-/issues/151 + # data and another that holds this data after compression. + # For tracking efficiency lines, these counters fluctuate for some reason. r' \| "Size of compressed data" .*', r' \| "Size of serialized data" .*' ]) -- GitLab From dd6f9ced093b366df179ad3f8c864d252c9303d0 Mon Sep 17 00:00:00 2001 From: sesen Date: Mon, 3 Oct 2022 18:48:59 +0200 Subject: [PATCH 5/7] exclude checksum check for Relations and CaloHypo packers --- Hlt/Moore/python/Moore/qmtest/exclusions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Hlt/Moore/python/Moore/qmtest/exclusions.py b/Hlt/Moore/python/Moore/qmtest/exclusions.py index 4edb606e0ff..dfb95df02ba 100644 --- a/Hlt/Moore/python/Moore/qmtest/exclusions.py +++ b/Hlt/Moore/python/Moore/qmtest/exclusions.py @@ -61,7 +61,10 @@ remove_known_fluctuating_counters = LineSkipper(regexps=[ # data and another that holds this data after compression. # For tracking efficiency lines, these counters fluctuate for some reason. r' \| "Size of compressed data" .*', - r' \| "Size of serialized data" .*' + r' \| "Size of serialized data" .*', + r"P2VRelationPacker\s+INFO Packed data checksum for .*", + r"PP2MCPRelationPacker\s+INFO Packed data checksum for .*", + r"CaloHypoPacker\s+INFO Packed data checksum for .*" ]) # In most cases everything interesting happens at finalize: counters, -- GitLab From ecc37ef4f6840a61a136084270eea569d4ccc019 Mon Sep 17 00:00:00 2001 From: sesen Date: Mon, 3 Oct 2022 18:51:57 +0200 Subject: [PATCH 6/7] remove fluctuating counters from checksum test --- .../tests/qmtest/test_hlt2_check_packed_data_checksums.qmt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt index 9e0cf24abec..eddf606d63a 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt @@ -29,7 +29,8 @@ countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, from Moore.qmtest.exclusions import skip_initialize, skip_scheduler, remove_known_fluctuating_counters from RecConf.QMTest.exclusions import preprocessor as RecPreprocessor -validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler) +validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler, + counter_preproc = remove_known_fluctuating_counters) -- GitLab From 3305d7fe82517f5c67cdac8fef0176311343253d Mon Sep 17 00:00:00 2001 From: Rosen Matev Date: Mon, 3 Oct 2022 23:30:36 +0200 Subject: [PATCH 7/7] Tweak exclusions --- .../qmtest/test_hlt2_check_packed_data_checksums.qmt | 11 +++++++++-- Hlt/Moore/python/Moore/qmtest/exclusions.py | 3 +-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt index eddf606d63a..105f0ff8a69 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt @@ -29,8 +29,15 @@ countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, from Moore.qmtest.exclusions import skip_initialize, skip_scheduler, remove_known_fluctuating_counters from RecConf.QMTest.exclusions import preprocessor as RecPreprocessor -validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler, - counter_preproc = remove_known_fluctuating_counters) + +# FIXME understand the reason for these fluctuations, see LHCb!151 +preprocessor = skip_initialize + RecPreprocessor + skip_scheduler + LineSkipper(regexps=[ + r"P2VRelationPacker\s+INFO Packed data checksum for .*", + r"CaloHypoPacker\s+INFO Packed data checksum for .*", +]) + +validateWithReference(preproc = preprocessor, + counter_preproc = remove_known_fluctuating_counters) diff --git a/Hlt/Moore/python/Moore/qmtest/exclusions.py b/Hlt/Moore/python/Moore/qmtest/exclusions.py index dfb95df02ba..e293dbef8d0 100644 --- a/Hlt/Moore/python/Moore/qmtest/exclusions.py +++ b/Hlt/Moore/python/Moore/qmtest/exclusions.py @@ -63,8 +63,7 @@ remove_known_fluctuating_counters = LineSkipper(regexps=[ r' \| "Size of compressed data" .*', r' \| "Size of serialized data" .*', r"P2VRelationPacker\s+INFO Packed data checksum for .*", - r"PP2MCPRelationPacker\s+INFO Packed data checksum for .*", - r"CaloHypoPacker\s+INFO Packed data checksum for .*" + r"CaloHypoPacker\s+INFO Packed data checksum for .*", ]) # In most cases everything interesting happens at finalize: counters, -- GitLab