From e05a32c2e76e0c0ecba4a64f16aee0c07f9e1bb6 Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Wed, 14 Sep 2022 22:15:23 +0200 Subject: [PATCH 1/3] CI: Profile factory creations and SQL queries via EventProf --- .gitlab-ci.yml | 2 + .gitlab/ci/rails/shared.gitlab-ci.yml | 4 +- scripts/rspec_helpers.sh | 3 ++ spec/spec_helper.rb | 3 ++ spec/support/event_prof.rb | 58 +++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 spec/support/event_prof.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a05caa79d749c..e3a160c323e016 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -206,6 +206,8 @@ variables: RSPEC_AUTO_EXPLAIN_LOG_PATH: auto_explain/auto_explain.ndjson.gz TMP_TEST_FOLDER: "${CI_PROJECT_DIR}/tmp/tests" TMP_TEST_GITLAB_WORKHORSE_PATH: "${TMP_TEST_FOLDER}/${GITLAB_WORKHORSE_FOLDER}" + EVENT_PROF_REPORT_PATH: rspec/event_prof.json + EVENT_PROF_TOP: 100000 ES_JAVA_OPTS: "-Xms256m -Xmx256m" ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200" diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index 9453bdd9b8db9c..4fa4c14c01730f 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -92,7 +92,9 @@ include: RECORD_DEPRECATIONS: "true" GEO_SECONDARY_PROXY: 0 SUCCESSFULLY_RETRIED_TEST_EXIT_CODE: 137 - EVENT_PROF: "sql.active_record" + EVENT_PROF: 'factory.create,sql.active_record' + EVENT_PROF_FORMAT: 'json' + EVENT_PROF_JSON_FILE: "event_prof_${CI_JOB_NAME}.json" needs: - !reference [.rspec-base-needs, needs] - job: "detect-tests" diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index afe41a06bdc056..e8645aae19b00e 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -47,6 +47,9 @@ function update_tests_metadata() { # in this pipeline, so that first_flaky_at for tests that are still flaky is maintained. scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH:-unknown_file}" + echo "{}" > "${EVENT_PROF_REPORT_PATH}" + scripts/merge-reports "${EVENT_PROF_REPORT_PATH}" rspec/test_prof/event_prof*.json + if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then if [[ -n "$RSPEC_PROFILING_PGSSLKEY" ]]; then chmod 0600 $RSPEC_PROFILING_PGSSLKEY diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c3e935f50edea4..26cb85f7c249f4 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -557,6 +557,9 @@ # Initialize FactoryDefault to use create_default helper TestProf::FactoryDefault.init +TestProf.configure do |config| + config.output_dir = ENV['CI'] ? 'rspec/test_prof' : 'tmp/test_prof' +end # Set the start of ID sequence for records initialized by `build_stubbed` to prevent conflicts FactoryBot::Strategy::Stub.next_id = 1_000_000_000 diff --git a/spec/support/event_prof.rb b/spec/support/event_prof.rb new file mode 100644 index 00000000000000..3bfd16ccec6678 --- /dev/null +++ b/spec/support/event_prof.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +return unless ENV['EVENT_PROF'] +return unless ENV['EVENT_PROF_FORMAT'] == 'json' + +require 'json' + +module Support + module EventProf + module JsonPrinter + def print + super + + # group_location => { + # event => result + # } + json = Hash.new { |hash, location| hash[location] = {} } + + @profiler.each do |profile| + collect_data(json, profile) + end + + write_json(json) + end + + private + + def collect_data(json, profile) + profile.results[:groups].each do |group| + metadata = group[:id].metadata + path = metadata[:file_path] + json[path][:description] = metadata[:description] + json[path][:type] = metadata[:type] || '?' + + json[path][profile.event] = { + time: group[:time], + run_time: group[:run_time], + count: group[:count] + } + end + end + + def write_json(json) + filename = ENV['EVENT_PROF_JSON_FILE'] || 'event_prof' + filename = filename.gsub(/\W/, '_') + filename = "#{filename}.json" + + path = TestProf.artifact_path(filename) + + File.write(path, JSON.generate(json)) # rubocop:disable Gitlab/Json + + log :info, "EventProf results written to #{path}" + end + end + end +end + +TestProf::EventProf::RSpecListener.prepend Support::EventProf::JsonPrinter -- GitLab From dd67751e74e5458f89c6ad5228ac3893896c3abd Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Fri, 23 Sep 2022 12:32:19 +0200 Subject: [PATCH 2/3] Track amount of examples per group --- spec/support/event_prof.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/support/event_prof.rb b/spec/support/event_prof.rb index 3bfd16ccec6678..adbeed7f75a0b7 100644 --- a/spec/support/event_prof.rb +++ b/spec/support/event_prof.rb @@ -31,6 +31,7 @@ def collect_data(json, profile) path = metadata[:file_path] json[path][:description] = metadata[:description] json[path][:type] = metadata[:type] || '?' + json[path][:examples] = group[:id].descendant_filtered_examples.size json[path][profile.event] = { time: group[:time], @@ -47,7 +48,7 @@ def write_json(json) path = TestProf.artifact_path(filename) - File.write(path, JSON.generate(json)) # rubocop:disable Gitlab/Json + File.write(path, JSON.generate(json)) # rubocop:disable Gitlab/Json -- This is fine. log :info, "EventProf results written to #{path}" end -- GitLab From 4e72f619f18a8ce796368b32a887842a7668bd0d Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Mon, 12 Aug 2024 17:36:24 +0200 Subject: [PATCH 3/3] Compile factory creation statistics via test-prof's FactoryProf JSON --- .gitlab-ci.yml | 1 + .gitlab/ci/rails/shared.gitlab-ci.yml | 1 + scripts/rspec_helpers.sh | 3 ++ spec/support/factory_prof.rb | 44 +++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 spec/support/factory_prof.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e3a160c323e016..a01c992410e207 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,6 +208,7 @@ variables: TMP_TEST_GITLAB_WORKHORSE_PATH: "${TMP_TEST_FOLDER}/${GITLAB_WORKHORSE_FOLDER}" EVENT_PROF_REPORT_PATH: rspec/event_prof.json EVENT_PROF_TOP: 100000 + FPROF_REPORT_PATH: rspec/factory_prof.json ES_JAVA_OPTS: "-Xms256m -Xmx256m" ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200" diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index 4fa4c14c01730f..3426a3cc247ad2 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -92,6 +92,7 @@ include: RECORD_DEPRECATIONS: "true" GEO_SECONDARY_PROXY: 0 SUCCESSFULLY_RETRIED_TEST_EXIT_CODE: 137 + FPROF: 'json' EVENT_PROF: 'factory.create,sql.active_record' EVENT_PROF_FORMAT: 'json' EVENT_PROF_JSON_FILE: "event_prof_${CI_JOB_NAME}.json" diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index e8645aae19b00e..501fbea75b0861 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -50,6 +50,9 @@ function update_tests_metadata() { echo "{}" > "${EVENT_PROF_REPORT_PATH}" scripts/merge-reports "${EVENT_PROF_REPORT_PATH}" rspec/test_prof/event_prof*.json + echo "{}" > "${FPROF_REPORT_PATH}" + scripts/merge-reports "${FPROF_REPORT_PATH}" rspec/test_prof/factory_prof_*.json + if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then if [[ -n "$RSPEC_PROFILING_PGSSLKEY" ]]; then chmod 0600 $RSPEC_PROFILING_PGSSLKEY diff --git a/spec/support/factory_prof.rb b/spec/support/factory_prof.rb new file mode 100644 index 00000000000000..a56c2e085fb18e --- /dev/null +++ b/spec/support/factory_prof.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +return unless ENV['FPROF'] == 'json' + +require 'json' + +module Support + module FactoryProfPrinter + class << self + include TestProf::Logging + + def dump(result, start_time:) # rubocop:disable Lint/UnusedMethodArgument -- really unused. + json = convert_stats(result) + write_json(json) + end + + private + + def convert_stats(result) + { job_name => result.stats } + end + + def write_json(json) + filename = "factory_prof_#{job_name}" + filename = filename.gsub(/\W/, '_') + filename = "#{filename}.json" + + path = TestProf.artifact_path(filename) + + File.write(path, JSON.generate(json)) # rubocop:disable Gitlab/Json -- This is fine. + + log :info, "FactoryProf results written to #{path}" + end + + def job_name + ENV['CI_JOB_NAME'] || '__main__' + end + end + end +end + +TestProf::FactoryProf.configure do |config| + config.printer = Support::FactoryProfPrinter +end -- GitLab