From bc2cc5732b6028105fdadbd41ba1b3a31d17b285 Mon Sep 17 00:00:00 2001 From: Denys Mishunov Date: Tue, 30 Sep 2025 22:26:36 +0200 Subject: [PATCH 1/4] feat: use glob pattern for files in node_modules cache --- .gitlab/ci/caching.gitlab-ci.yml | 20 ++++++++++++++++ .gitlab/ci/global.gitlab-ci.yml | 6 +---- .gitlab/ci/rules.gitlab-ci.yml | 6 +++++ scripts/lib/node_modules_sha.rb | 41 ++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 scripts/lib/node_modules_sha.rb diff --git a/.gitlab/ci/caching.gitlab-ci.yml b/.gitlab/ci/caching.gitlab-ci.yml index 0b20b86ab7c6a9..8dfb76ac86fc3d 100644 --- a/.gitlab/ci/caching.gitlab-ci.yml +++ b/.gitlab/ci/caching.gitlab-ci.yml @@ -102,6 +102,23 @@ cache:workhorse-hash: reports: dotenv: build.env +#================================= +# node_modules +#================================= + +cache:node-modules-hash: + extends: + - .cache-pull-push-base + - .with-alpine-ruby-image + - .caching:rules:update-cache-on-code-changes + script: + - echo "GLCI_NODE_MODULES_HASH=$(ruby scripts/lib/node_modules_sha.rb)" > build.env + - echo "Calculated node modules checksum - $(cat build.env)" + artifacts: + reports: + dotenv: build.env + + #================================= # Frontend #================================= @@ -118,6 +135,9 @@ cache:assets-hash: dotenv: build.env allow_failure: false +#================================= +# FE Islands +#================================= cache:frontend-islands-hash: extends: - .cache-pull-push-base diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 575d9540823cc3..0930dad2fe2d36 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -444,11 +444,7 @@ policy: pull-push .node-modules-cache: &node-modules-cache - key: - files: - - yarn.lock - - ee/frontend_islands/apps/duo_next/yarn.lock - prefix: node-modules-${BUILD_OS}-${OS_VERSION}-${NODE_ENV}-node-${NODE_VERSION} + key: "node-modules-${BUILD_OS}-${OS_VERSION}-${NODE_ENV}-node-${NODE_VERSION}-${GLCI_NODE_MODULES_HASH}" paths: - node_modules/ - ee/frontend_islands/apps/duo_next/node_modules/ diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 7ae8a1adce9b03..346a6da5aae6db 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -646,6 +646,12 @@ - "{,ee/,jh/}spec/**/*" - "scripts/**/*" - "{,spec/}tooling/**/*" + # FE island changes + - "ee/frontend_islands/apps/**/.{gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - "ee/frontend_islands/apps/**/{package.json,yarn.lock}" + - "ee/frontend_islands/apps/**/tsconfig*.json" + - "ee/frontend_islands/apps/**/components.json" + - "ee/frontend_islands/apps/**/*.config.{js,ts,cjs,mjs}" # .code-patterns + .qa-patterns .code-qa-patterns: &code-qa-patterns diff --git a/scripts/lib/node_modules_sha.rb b/scripts/lib/node_modules_sha.rb new file mode 100644 index 00000000000000..b78508273db81c --- /dev/null +++ b/scripts/lib/node_modules_sha.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "digest" + +module NodeModulesSha + # Node.js dependency files that impact node_modules installation + NODE_MODULES_FILES = %w[ + yarn.lock + package.json + ee/frontend_islands/apps/duo_next/yarn.lock + ee/frontend_islands/apps/duo_next/package.json + ].freeze + + class << self + def cached_node_modules_sha256 + @cached_node_modules_sha256 ||= ENV + .fetch('GLCI_NODE_MODULES_HASH_FILE', 'cached-node-modules-hash.txt') + .then do |file| + next 'missing!' unless File.exist?(file) + + File.read(file).strip + end + end + + def sha256_of_node_modules_impacting_compilation + node_modules_sha256 = node_modules_impacting_compilation.map do |file| + Digest::SHA256.file(file).hexdigest + end.join + + Digest::SHA256.hexdigest(node_modules_sha256) + end + + private + + def node_modules_impacting_compilation + NODE_MODULES_FILES.select { |file| File.exist?(file) && File.file?(file) }.sort + end + end +end + +printf NodeModulesSha.sha256_of_node_modules_impacting_compilation if __FILE__ == $PROGRAM_NAME -- GitLab From 79e53466eed0f88a3938c35a02ca15663b4e1cdc Mon Sep 17 00:00:00 2001 From: Denys Mishunov Date: Wed, 1 Oct 2025 00:10:42 +0200 Subject: [PATCH 2/4] refactor: removed unused method --- scripts/lib/node_modules_sha.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/lib/node_modules_sha.rb b/scripts/lib/node_modules_sha.rb index b78508273db81c..cfe73c86cf1099 100644 --- a/scripts/lib/node_modules_sha.rb +++ b/scripts/lib/node_modules_sha.rb @@ -12,16 +12,6 @@ module NodeModulesSha ].freeze class << self - def cached_node_modules_sha256 - @cached_node_modules_sha256 ||= ENV - .fetch('GLCI_NODE_MODULES_HASH_FILE', 'cached-node-modules-hash.txt') - .then do |file| - next 'missing!' unless File.exist?(file) - - File.read(file).strip - end - end - def sha256_of_node_modules_impacting_compilation node_modules_sha256 = node_modules_impacting_compilation.map do |file| Digest::SHA256.file(file).hexdigest -- GitLab From be9458c2de17855571138679ab14cad40b545569 Mon Sep 17 00:00:00 2001 From: Denys Mishunov Date: Wed, 1 Oct 2025 00:47:48 +0200 Subject: [PATCH 3/4] fix: added the cache:node-modules-hash as dependency for jobs --- .gitlab/ci/global.gitlab-ci.yml | 8 ++++++++ .gitlab/ci/test-metadata.gitlab-ci.yml | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 0930dad2fe2d36..5fb2f539d8f92d 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -538,6 +538,7 @@ .static-analysis-cache: extends: .cache-analytics + needs: ["cache:node-modules-hash"] cache: - !reference [.ruby-gems-cache] - *node-modules-cache @@ -554,6 +555,7 @@ - *rubocop-cache-push .ruby-node-cache: + needs: ["cache:node-modules-hash"] cache: - !reference [.ruby-gems-cache] - *node-modules-cache @@ -568,10 +570,12 @@ policy: pull-push .yarn-cache: + needs: ["cache:node-modules-hash"] cache: - *node-modules-cache .yarn-cache-pull-push: + needs: ["cache:node-modules-hash"] cache: <<: *node-modules-cache policy: pull-push @@ -579,6 +583,7 @@ .assets-compile-cache: extends: - .cache-analytics + needs: ["cache:node-modules-hash"] cache: - !reference [.ruby-gems-cache] - *node-modules-cache @@ -588,6 +593,7 @@ .assets-compile-cache-pull: # this cache is used only for gdk:compile-test-assets which should only pull, never push extends: - .cache-analytics + needs: ["cache:node-modules-hash"] cache: - !reference [.ruby-gems-cache] - *node-modules-cache @@ -596,11 +602,13 @@ policy: pull .storybook-yarn-cache: + needs: ["cache:node-modules-hash"] cache: - *node-modules-cache - *storybook-node-modules-cache .storybook-yarn-cache-pull-push: + needs: ["cache:node-modules-hash"] cache: - *node-modules-cache - *storybook-node-modules-cache-pull-push diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index d46ffcca831c60..51fe8c2d144e4e 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -77,6 +77,10 @@ export-predictive-test-metrics: - .predictive:rules:default variables: GLCI_PREDICTIVE_TEST_METRICS_OUTPUT_DIR: "tmp/predictive_tests" + needs: + - cache:node-modules-hash + - graphql-schema-dump + - retrieve-tests-metadata dependencies: # required to generate predictive jest test list - graphql-schema-dump -- GitLab From 16b1a8cf13907b388dbcb69cb1058f633237a994 Mon Sep 17 00:00:00 2001 From: Denys Mishunov Date: Wed, 1 Oct 2025 02:06:06 +0200 Subject: [PATCH 4/4] fix: update all jobs that need node-modules cache --- .gitlab/ci/docs.gitlab-ci.yml | 3 ++- .gitlab/ci/frontend.gitlab-ci.yml | 10 ++++++++++ .../ci/includes/gitlab-com/danger-review.gitlab-ci.yml | 1 + .gitlab/ci/preflight.gitlab-ci.yml | 4 ++++ .gitlab/ci/static-analysis.gitlab-ci.yml | 3 +++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index a6fa77557a9b8c..f2e149f493797c 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -70,7 +70,8 @@ docs-lint mermaid: - .docs-markdown-lint-image - .yarn-cache stage: lint - needs: [] + needs: + - job: cache:node-modules-hash script: - source ./scripts/utils.sh - yarn_install_script diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 3e3b24573426ea..f02e5dfaeb0a9d 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -26,6 +26,7 @@ WEBPACK_COMPILE_LOG_PATH: "tmp/webpack-output.log" stage: prepare needs: + - cache:node-modules-hash - cache:assets-hash - job: cache:frontend-islands-hash optional: true @@ -99,6 +100,7 @@ build-vite-prod: - .frontend:rules:compile-production-assets stage: prepare needs: + - job: cache:node-modules-hash - job: cache:node-modules optional: true script: @@ -255,6 +257,7 @@ upload-graphql-schema-dump: - !reference [.yarn-cache, cache] - !reference [.frontend-islands-cache-pull] needs: + - job: cache:node-modules-hash - job: cache:frontend-islands-hash optional: true @@ -587,6 +590,7 @@ coverage-frontend: - .frontend:rules:coverage-frontend needs: - !reference [.repo-from-artifacts, needs] + - job: cache:node-modules-hash - job: "jest" optional: true - job: "jest-with-fixtures" @@ -620,6 +624,10 @@ webpack-dev-server: - .repo-from-artifacts - .frontend:rules:default-frontend-jobs stage: test-frontend + needs: + - job: cache:node-modules-hash + - job: clone-gitlab-repo + optional: true variables: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" @@ -645,6 +653,7 @@ compile-storybook: - !reference [.repo-from-artifacts, needs] - !reference [.with-fixtures-needs, needs] - !reference [.with-graphql-schema-dump-needs, needs] + - job: cache:node-modules-hash - job: cache:frontend-islands-hash optional: true artifacts: @@ -669,6 +678,7 @@ test-storybook: - !reference [.repo-from-artifacts, needs] - !reference [.with-fixtures-needs, needs] - !reference [.with-graphql-schema-dump-needs, needs] + - job: cache:node-modules-hash - job: cache:frontend-islands-hash optional: true allow_failure: true diff --git a/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml b/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml index 4f656505562995..aace5a7828c278 100644 --- a/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml +++ b/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml @@ -16,6 +16,7 @@ danger-review: needs: - job: cache:ruby-gems optional: true + - job: cache:node-modules-hash - job: cache:node-modules optional: true variables: diff --git a/.gitlab/ci/preflight.gitlab-ci.yml b/.gitlab/ci/preflight.gitlab-ci.yml index cf997340b49b52..e969e223115bb1 100644 --- a/.gitlab/ci/preflight.gitlab-ci.yml +++ b/.gitlab/ci/preflight.gitlab-ci.yml @@ -79,6 +79,10 @@ bundle-size-review: - .repo-from-artifacts - .frontend:rules:bundle-size-review stage: preflight + needs: + - job: clone-gitlab-repo + optional: true + - job: cache:node-modules-hash variables: SETUP_DB: "false" script: diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml index 84873dca5a3334..7e77018691a7ba 100644 --- a/.gitlab/ci/static-analysis.gitlab-ci.yml +++ b/.gitlab/ci/static-analysis.gitlab-ci.yml @@ -21,6 +21,7 @@ static-analysis: needs: - job: cache:ruby-gems optional: true + - job: cache:node-modules-hash - job: cache:node-modules optional: true - job: db:setup pg16 @@ -122,6 +123,8 @@ eslint-docs: - .static-analysis-base - .yarn-cache - .frontend:rules:default-frontend-jobs-with-docs-changes + needs: + - job: cache:node-modules-hash variables: USE_BUNDLE_INSTALL: "false" script: -- GitLab