diff --git a/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py b/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py index a034c8b2903fc68157cbb11ccda9528e7869847b..b24f1d04d2571feb9c6c30f3d53ea69085003d36 100644 --- a/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py +++ b/Hlt/Hlt2Conf/tests/options/hlt2_checksum_packed_data.py @@ -12,8 +12,7 @@ """ from Moore import options, run_moore -from PyConf.Algorithms import HltPackedBufferWriter -from Gaudi.Configuration import DEBUG +from Moore.persistence.packing import pack_stream_objects from RecoConf.global_tools import stateProvider_with_simplified_geom from RecoConf.reconstruction_objects import reconstruction @@ -29,6 +28,6 @@ def make_lines(): public_tools = [stateProvider_with_simplified_geom()] -with reconstruction.bind(from_file=False), HltPackedBufferWriter.bind( - OutputLevel=DEBUG, EnableChecksum=True): +with reconstruction.bind(from_file=False), pack_stream_objects.bind( + enable_checksum=True): config = run_moore(options, make_lines, public_tools) diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt index 17d434bf209f137f2f7dbc7204bdda2accdd41a1..105f0ff8a6991081a3cbb32b16b09a00ae08e250 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_check_packed_data_checksums.qmt @@ -29,17 +29,16 @@ countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, from Moore.qmtest.exclusions import skip_initialize, skip_scheduler, remove_known_fluctuating_counters from RecConf.QMTest.exclusions import preprocessor as RecPreprocessor -validateWithReference(preproc = skip_initialize + RecPreprocessor + skip_scheduler, counter_preproc = remove_known_fluctuating_counters) - +# FIXME understand the reason for these fluctuations, see LHCb!151 +preprocessor = skip_initialize + RecPreprocessor + skip_scheduler + LineSkipper(regexps=[ + r"P2VRelationPacker\s+INFO Packed data checksum for .*", + r"CaloHypoPacker\s+INFO Packed data checksum for .*", +]) - - - .* - +validateWithReference(preproc = preprocessor, + counter_preproc = remove_known_fluctuating_counters) + diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt index d6a77f543e1ee32bf88528f185f83c9fb745bded..c74f1bbdb82cbafdd5286163f616b936ef08fd8a 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_noUT_trackefflines.qmt @@ -23,8 +23,8 @@ Make sure the HLT2 line example configures and runs without errors 1200 -from Moore.qmtest.exclusions import ref_preprocessor -validateWithReference(preproc = ref_preprocessor) +from Moore.qmtest.exclusions import ref_preprocessor, remove_known_fluctuating_counters +validateWithReference(preproc = ref_preprocessor, counter_preproc = remove_known_fluctuating_counters) from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt index 95c8d4513fd17d084484c28853849cafea260e7a..5739843b3b47aec692032560fb338378ddfba83e 100644 --- a/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt +++ b/Hlt/Hlt2Conf/tests/qmtest/test_hlt2_trackefflines.qmt @@ -23,8 +23,8 @@ Make sure the HLT2 line example configures and runs without errors 1200 -from Moore.qmtest.exclusions import ref_preprocessor -validateWithReference(preproc = ref_preprocessor) +from Moore.qmtest.exclusions import ref_preprocessor, remove_known_fluctuating_counters +validateWithReference(preproc = ref_preprocessor, counter_preproc = remove_known_fluctuating_counters) from Moore.qmtest.exclusions import remove_known_warnings countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, diff --git a/Hlt/Moore/python/Moore/persistence/packing.py b/Hlt/Moore/python/Moore/persistence/packing.py index a8024185d065c7ac04a41e78f14cc62fdebe05a8..569be20223f366213412e4485c6204494af445a5 100644 --- a/Hlt/Moore/python/Moore/persistence/packing.py +++ b/Hlt/Moore/python/Moore/persistence/packing.py @@ -27,7 +27,8 @@ def pack_stream_objects(stream, inputs, encoding_key, source_id, - enable_check=False): + enable_check=False, + enable_checksum=False): """Return CF node that packs and serialises a set of containers to a raw bank. Args: stream (str): TES root containing objects to be persisted. @@ -46,6 +47,7 @@ def pack_stream_objects(stream, packer = p( InputName=[force_location(loc) for loc in inputs[t]], EnableCheck=enable_check, + EnableChecksum=enable_checksum, EncodingKey=encoding_key) packers += [packer] diff --git a/Hlt/Moore/python/Moore/qmtest/exclusions.py b/Hlt/Moore/python/Moore/qmtest/exclusions.py index 0890d70e5805d2c4d692cff54da94dfe40c8a61a..e293dbef8d005663e10d9b03f4d2a64082d26ed7 100644 --- a/Hlt/Moore/python/Moore/qmtest/exclusions.py +++ b/Hlt/Moore/python/Moore/qmtest/exclusions.py @@ -58,11 +58,12 @@ remove_known_fluctuating_counters = LineSkipper(regexps=[ # reconstruction tests based on HLT2 line content r' \| "# loaded from (?:CACHE|PYTHON)" .*', # The HltPackedDataWriter uses two data buffers: one that holds uncompressed - # data and another that holds this data after compression. For some reason, - # the size of the compressed data buffer fluctuates slightly (~0.001%) between runs. - # The same does not happen for the uncompressed data buffer. - # See https://gitlab.cern.ch/lhcb/LHCb/-/issues/151 - r' \| "Size of compressed data" .*' + # data and another that holds this data after compression. + # For tracking efficiency lines, these counters fluctuate for some reason. + r' \| "Size of compressed data" .*', + r' \| "Size of serialized data" .*', + r"P2VRelationPacker\s+INFO Packed data checksum for .*", + r"CaloHypoPacker\s+INFO Packed data checksum for .*", ]) # In most cases everything interesting happens at finalize: counters,