From 9cd756e3a34822998ba24431efdccdb9aa5e9a71 Mon Sep 17 00:00:00 2001 From: Chad Woolley Date: Tue, 27 Aug 2024 00:34:07 -0700 Subject: [PATCH 1/6] Delete remote_development_agent_configs table - See https://gitlab.com/gitlab-org/gitlab/-/issues/480767 --- ...delete_remote_development_agent_configs.rb | 51 +++++++++++++++++++ db/schema_migrations/20240826000001 | 1 + db/structure.sql | 47 ----------------- scripts/remote_development/run-e2e-tests.sh | 9 +++- 4 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 db/migrate/20240826000001_delete_remote_development_agent_configs.rb create mode 100644 db/schema_migrations/20240826000001 diff --git a/db/migrate/20240826000001_delete_remote_development_agent_configs.rb b/db/migrate/20240826000001_delete_remote_development_agent_configs.rb new file mode 100644 index 00000000000000..362bec33ebd27b --- /dev/null +++ b/db/migrate/20240826000001_delete_remote_development_agent_configs.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +class DeleteRemoteDevelopmentAgentConfigs < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone "17.6" + + TABLE_NAME = :remote_development_agent_configs + + def constraint_1_name + check_constraint_name TABLE_NAME, :default_max_hours_before_termination, "max_size_1_year" + end + + def constraint_2_name + check_constraint_name TABLE_NAME, :max_hours_before_termination_limit, "max_size_1_year" + end + + def up + drop_table TABLE_NAME # rubocop:disable Migration/DropTable -- We have already migrated this table to workspaces_agent_configs - see https://gitlab.com/gitlab-org/gitlab/-/issues/480135 + end + + # noinspection RubyResolve -- RubyMine doesn't resolve t.bigint. TODO: Open ticket and link on https://handbook.gitlab.com/handbook/tools-and-tips/editors-and-ides/jetbrains-ides/tracked-jetbrains-issues/ + def down + create_table TABLE_NAME do |t| + t.timestamps_with_timezone null: false + t.bigint "cluster_agent_id", null: false + t.bigint "workspaces_quota", default: -1, null: false + t.bigint "workspaces_per_user_quota", default: -1, null: false + t.bigint "project_id" + t.column "default_max_hours_before_termination", :smallint, default: 24, null: false + t.column "max_hours_before_termination_limit", :smallint, default: 120, null: false + t.boolean "enabled", null: false + t.boolean "network_policy_enabled", default: true, null: false + t.text "dns_zone", limit: 256, null: false + # Kubernetes namespaces are limited to 63 characters + t.text "gitlab_workspaces_proxy_namespace", limit: 63, default: "gitlab-workspaces", null: false + t.jsonb "network_policy_egress", + default: [{ "allow" => "0.0.0.0/0", "except" => %w[10.0.0.0/8 172.16.0.0/12 192.168.0.0/16] }], null: false + t.jsonb "default_resources_per_workspace_container", default: {}, null: false + t.jsonb "max_resources_per_workspace", default: {}, null: false + + t.index :cluster_agent_id, unique: true, name: "index_remote_development_agent_configs_on_unique_agent_id" + t.index :project_id + end + + add_concurrent_foreign_key TABLE_NAME, :projects, column: :project_id, on_delete: :cascade + add_concurrent_foreign_key TABLE_NAME, :cluster_agents, column: :cluster_agent_id, on_delete: :cascade + add_check_constraint TABLE_NAME, "default_max_hours_before_termination <= 8760", constraint_1_name + add_check_constraint TABLE_NAME, "max_hours_before_termination_limit <= 8760", constraint_2_name + end +end diff --git a/db/schema_migrations/20240826000001 b/db/schema_migrations/20240826000001 new file mode 100644 index 00000000000000..a1deca5e2913e4 --- /dev/null +++ b/db/schema_migrations/20240826000001 @@ -0,0 +1 @@ +dea9cad933504e79291dfb159017376250cac80c6fd29786f9db8cd0a470ce0b \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index fd329190b34e01..4599907dedb246 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19067,36 +19067,6 @@ CREATE SEQUENCE releases_id_seq ALTER SEQUENCE releases_id_seq OWNED BY releases.id; -CREATE TABLE remote_development_agent_configs ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - cluster_agent_id bigint NOT NULL, - enabled boolean NOT NULL, - dns_zone text NOT NULL, - network_policy_enabled boolean DEFAULT true NOT NULL, - gitlab_workspaces_proxy_namespace text DEFAULT 'gitlab-workspaces'::text NOT NULL, - network_policy_egress jsonb DEFAULT '[{"allow": "0.0.0.0/0", "except": ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]}]'::jsonb NOT NULL, - default_resources_per_workspace_container jsonb DEFAULT '{}'::jsonb NOT NULL, - max_resources_per_workspace jsonb DEFAULT '{}'::jsonb NOT NULL, - workspaces_quota bigint DEFAULT '-1'::integer NOT NULL, - workspaces_per_user_quota bigint DEFAULT '-1'::integer NOT NULL, - project_id bigint, - default_max_hours_before_termination smallint DEFAULT 24 NOT NULL, - max_hours_before_termination_limit smallint DEFAULT 120 NOT NULL, - CONSTRAINT check_72947a4495 CHECK ((char_length(gitlab_workspaces_proxy_namespace) <= 63)), - CONSTRAINT check_9f5cd54d1c CHECK ((char_length(dns_zone) <= 256)) -); - -CREATE SEQUENCE remote_development_agent_configs_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE remote_development_agent_configs_id_seq OWNED BY remote_development_agent_configs.id; - CREATE TABLE remote_development_namespace_cluster_agent_mappings ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -24203,8 +24173,6 @@ ALTER TABLE ONLY release_links ALTER COLUMN id SET DEFAULT nextval('release_link ALTER TABLE ONLY releases ALTER COLUMN id SET DEFAULT nextval('releases_id_seq'::regclass); -ALTER TABLE ONLY remote_development_agent_configs ALTER COLUMN id SET DEFAULT nextval('remote_development_agent_configs_id_seq'::regclass); - ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings ALTER COLUMN id SET DEFAULT nextval('remote_development_namespace_cluster_agent_mappings_id_seq'::regclass); ALTER TABLE ONLY remote_mirrors ALTER COLUMN id SET DEFAULT nextval('remote_mirrors_id_seq'::regclass); @@ -26909,9 +26877,6 @@ ALTER TABLE releases ALTER TABLE ONLY releases ADD CONSTRAINT releases_pkey PRIMARY KEY (id); -ALTER TABLE ONLY remote_development_agent_configs - ADD CONSTRAINT remote_development_agent_configs_pkey PRIMARY KEY (id); - ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings ADD CONSTRAINT remote_development_namespace_cluster_agent_mappings_pkey PRIMARY KEY (id); @@ -32427,10 +32392,6 @@ CREATE UNIQUE INDEX index_releases_on_project_tag_unique ON releases USING btree CREATE INDEX index_releases_on_released_at ON releases USING btree (released_at); -CREATE INDEX index_remote_development_agent_configs_on_project_id ON remote_development_agent_configs USING btree (project_id); - -CREATE UNIQUE INDEX index_remote_development_agent_configs_on_unique_agent_id ON remote_development_agent_configs USING btree (cluster_agent_id); - CREATE INDEX index_remote_mirrors_on_last_successful_update_at ON remote_mirrors USING btree (last_successful_update_at); CREATE INDEX index_remote_mirrors_on_project_id ON remote_mirrors USING btree (project_id); @@ -35711,8 +35672,6 @@ CREATE TRIGGER trigger_2dafd0d13605 BEFORE INSERT OR UPDATE ON pages_domain_acme CREATE TRIGGER trigger_30209d0fba3e BEFORE INSERT OR UPDATE ON alert_management_alert_user_mentions FOR EACH ROW EXECUTE FUNCTION trigger_30209d0fba3e(); -CREATE TRIGGER trigger_3691f9f6a69f BEFORE INSERT OR UPDATE ON remote_development_agent_configs FOR EACH ROW EXECUTE FUNCTION trigger_3691f9f6a69f(); - CREATE TRIGGER trigger_388de55cd36c BEFORE INSERT OR UPDATE ON ci_builds_runner_session FOR EACH ROW EXECUTE FUNCTION trigger_388de55cd36c(); CREATE TRIGGER trigger_38bfee591e40 BEFORE INSERT OR UPDATE ON dependency_proxy_blob_states FOR EACH ROW EXECUTE FUNCTION trigger_38bfee591e40(); @@ -36050,9 +36009,6 @@ ALTER TABLE ONLY merge_request_assignment_events ALTER TABLE ONLY catalog_resource_component_last_usages ADD CONSTRAINT fk_094c686785 FOREIGN KEY (component_project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE ONLY remote_development_agent_configs - ADD CONSTRAINT fk_0a3c0ada56 FOREIGN KEY (cluster_agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE; - ALTER TABLE ONLY dast_sites ADD CONSTRAINT fk_0a57f2271b FOREIGN KEY (dast_site_validation_id) REFERENCES dast_site_validations(id) ON DELETE SET NULL; @@ -36662,9 +36618,6 @@ ALTER TABLE p_ci_builds ALTER TABLE ONLY ai_conversation_messages ADD CONSTRAINT fk_68774ec148 FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; -ALTER TABLE ONLY remote_development_agent_configs - ADD CONSTRAINT fk_6a09894a0f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; - ALTER TABLE ONLY merge_requests ADD CONSTRAINT fk_6a5165a692 FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; diff --git a/scripts/remote_development/run-e2e-tests.sh b/scripts/remote_development/run-e2e-tests.sh index 55650c22c70265..0877509bfa3fba 100755 --- a/scripts/remote_development/run-e2e-tests.sh +++ b/scripts/remote_development/run-e2e-tests.sh @@ -8,11 +8,16 @@ # # For details on how to run this, see the documentation comments at the top of # qa/qa/specs/features/ee/browser_ui/3_create/remote_development/with_prerequisite_done/workspace_actions_with_prerequisite_done_spec.rb -# Running the E2E specs always creates a personal access token for the logged in user, This can be skipped by passing the environment -# variable GITLAB_QA_ACCESS_TOKEN with PAT of the logged in user DEFAULT_PASSWORD='5iveL!fe' +if [[ -z "${GITLAB_QA_ACCESS_TOKEN}" ]]; then + echo + echo "NOTE: GITLAB_QA_ACCESS_TOKEN is not set. This causes the E2E spec to create a personal access token for the logged in user, which results in a slower test run." + echo " This can be skipped by passing the environment variable GITLAB_QA_ACCESS_TOKEN with PAT of the logged in user" + echo +fi + export WEBDRIVER_HEADLESS="${WEBDRIVER_HEADLESS:-0}" export GITLAB_USERNAME="${GITLAB_USERNAME:-root}" export GITLAB_INITIAL_ROOT_PASSWORD="${GITLAB_INITIAL_ROOT_PASSWORD:-${DEFAULT_PASSWORD}}" -- GitLab From 30fe563a9d5d6b2c4c12ddfb1850675719211809 Mon Sep 17 00:00:00 2001 From: Daniyal Arshad Date: Wed, 23 Oct 2024 12:39:41 -0400 Subject: [PATCH 2/6] Finalize migration and update migration timestamp for dropping table --- ...elete_remote_development_agent_configs.rb} | 0 ...l_root_namespace_cluster_agent_mappings.rb | 23 +++++++++++++++++++ db/schema_migrations/20240826000001 | 1 - db/schema_migrations/20241022220115 | 1 + db/schema_migrations/20241022230950 | 1 + 5 files changed, 25 insertions(+), 1 deletion(-) rename db/migrate/{20240826000001_delete_remote_development_agent_configs.rb => 20241022230950_delete_remote_development_agent_configs.rb} (100%) create mode 100644 db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb delete mode 100644 db/schema_migrations/20240826000001 create mode 100644 db/schema_migrations/20241022220115 create mode 100644 db/schema_migrations/20241022230950 diff --git a/db/migrate/20240826000001_delete_remote_development_agent_configs.rb b/db/migrate/20241022230950_delete_remote_development_agent_configs.rb similarity index 100% rename from db/migrate/20240826000001_delete_remote_development_agent_configs.rb rename to db/migrate/20241022230950_delete_remote_development_agent_configs.rb diff --git a/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb b/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb new file mode 100644 index 00000000000000..453e31cc7c23ce --- /dev/null +++ b/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class FinalizeBackfillRootNamespaceClusterAgentMappings < Gitlab::Database::Migration[2.2] + milestone '17.6' + MIGRATION = 'BackfillRootNamespaceClusterAgentMappings' + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :remote_development_agent_configs, + column_name: :id, + job_arguments: [], + finalize: true + ) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20240826000001 b/db/schema_migrations/20240826000001 deleted file mode 100644 index a1deca5e2913e4..00000000000000 --- a/db/schema_migrations/20240826000001 +++ /dev/null @@ -1 +0,0 @@ -dea9cad933504e79291dfb159017376250cac80c6fd29786f9db8cd0a470ce0b \ No newline at end of file diff --git a/db/schema_migrations/20241022220115 b/db/schema_migrations/20241022220115 new file mode 100644 index 00000000000000..c519dd35f9a673 --- /dev/null +++ b/db/schema_migrations/20241022220115 @@ -0,0 +1 @@ +7dcbe2dc1e5d570dc677533cb4daa4ef7381314512ec9323b98d6e586f83ef7f \ No newline at end of file diff --git a/db/schema_migrations/20241022230950 b/db/schema_migrations/20241022230950 new file mode 100644 index 00000000000000..ddff0f96b61c02 --- /dev/null +++ b/db/schema_migrations/20241022230950 @@ -0,0 +1 @@ +8752e7c7b03b65538fc33cf109f689d5abe5b1a7816d0b6c439d07de9e142c0a \ No newline at end of file -- GitLab From 08225602d7c5b7016750f9ec572b2d69d693c161 Mon Sep 17 00:00:00 2001 From: Daniyal Arshad Date: Wed, 23 Oct 2024 16:56:01 -0400 Subject: [PATCH 3/6] Revert finalization and update milestone for drop table migration --- ...delete_remote_development_agent_configs.rb | 2 +- ...l_root_namespace_cluster_agent_mappings.rb | 23 ------------------- db/schema_migrations/20241022220115 | 1 - 3 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb delete mode 100644 db/schema_migrations/20241022220115 diff --git a/db/migrate/20241022230950_delete_remote_development_agent_configs.rb b/db/migrate/20241022230950_delete_remote_development_agent_configs.rb index 362bec33ebd27b..69147d08fadd8f 100644 --- a/db/migrate/20241022230950_delete_remote_development_agent_configs.rb +++ b/db/migrate/20241022230950_delete_remote_development_agent_configs.rb @@ -3,7 +3,7 @@ class DeleteRemoteDevelopmentAgentConfigs < Gitlab::Database::Migration[2.2] disable_ddl_transaction! - milestone "17.6" + milestone "17.7" TABLE_NAME = :remote_development_agent_configs diff --git a/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb b/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb deleted file mode 100644 index 453e31cc7c23ce..00000000000000 --- a/db/post_migrate/20241022220115_finalize_backfill_root_namespace_cluster_agent_mappings.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class FinalizeBackfillRootNamespaceClusterAgentMappings < Gitlab::Database::Migration[2.2] - milestone '17.6' - MIGRATION = 'BackfillRootNamespaceClusterAgentMappings' - disable_ddl_transaction! - - restrict_gitlab_migration gitlab_schema: :gitlab_main_cell - - def up - ensure_batched_background_migration_is_finished( - job_class_name: MIGRATION, - table_name: :remote_development_agent_configs, - column_name: :id, - job_arguments: [], - finalize: true - ) - end - - def down - # no-op - end -end diff --git a/db/schema_migrations/20241022220115 b/db/schema_migrations/20241022220115 deleted file mode 100644 index c519dd35f9a673..00000000000000 --- a/db/schema_migrations/20241022220115 +++ /dev/null @@ -1 +0,0 @@ -7dcbe2dc1e5d570dc677533cb4daa4ef7381314512ec9323b98d6e586f83ef7f \ No newline at end of file -- GitLab From da9722f24dd25b3d3dd3eb1569af43065a429d57 Mon Sep 17 00:00:00 2001 From: Daniyal Arshad Date: Wed, 23 Oct 2024 16:57:52 -0400 Subject: [PATCH 4/6] Move dictionary to deleted_tables directory --- .../{ => deleted_tables}/remote_development_agent_configs.yml | 2 ++ 1 file changed, 2 insertions(+) rename db/docs/{ => deleted_tables}/remote_development_agent_configs.yml (86%) diff --git a/db/docs/remote_development_agent_configs.yml b/db/docs/deleted_tables/remote_development_agent_configs.yml similarity index 86% rename from db/docs/remote_development_agent_configs.yml rename to db/docs/deleted_tables/remote_development_agent_configs.yml index 4b50ff50326006..77414709e29237 100644 --- a/db/docs/remote_development_agent_configs.yml +++ b/db/docs/deleted_tables/remote_development_agent_configs.yml @@ -19,3 +19,5 @@ desired_sharding_key: belongs_to: agent desired_sharding_key_migration_job_name: BackfillRemoteDevelopmentAgentConfigsProjectId table_size: small +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/164032 +removed_in_milestone: '17.7' -- GitLab From b027cacbfcf5d6a5f3e878926865b6c405dcee86 Mon Sep 17 00:00:00 2001 From: Daniyal Arshad Date: Wed, 20 Nov 2024 15:42:39 -0500 Subject: [PATCH 5/6] Update migration timestamp --- ...=> 20241120200334_delete_remote_development_agent_configs.rb} | 0 db/schema_migrations/20241022230950 | 1 - db/schema_migrations/20241120200334 | 1 + 3 files changed, 1 insertion(+), 1 deletion(-) rename db/migrate/{20241022230950_delete_remote_development_agent_configs.rb => 20241120200334_delete_remote_development_agent_configs.rb} (100%) delete mode 100644 db/schema_migrations/20241022230950 create mode 100644 db/schema_migrations/20241120200334 diff --git a/db/migrate/20241022230950_delete_remote_development_agent_configs.rb b/db/migrate/20241120200334_delete_remote_development_agent_configs.rb similarity index 100% rename from db/migrate/20241022230950_delete_remote_development_agent_configs.rb rename to db/migrate/20241120200334_delete_remote_development_agent_configs.rb diff --git a/db/schema_migrations/20241022230950 b/db/schema_migrations/20241022230950 deleted file mode 100644 index ddff0f96b61c02..00000000000000 --- a/db/schema_migrations/20241022230950 +++ /dev/null @@ -1 +0,0 @@ -8752e7c7b03b65538fc33cf109f689d5abe5b1a7816d0b6c439d07de9e142c0a \ No newline at end of file diff --git a/db/schema_migrations/20241120200334 b/db/schema_migrations/20241120200334 new file mode 100644 index 00000000000000..72c540bf798da0 --- /dev/null +++ b/db/schema_migrations/20241120200334 @@ -0,0 +1 @@ +fa1ef5ccdbf45c4a0cb7848998337f497d22da165a836c1f841bd21bba7d81d0 \ No newline at end of file -- GitLab From 38201404eb797d60855a9ee4560230e57f66ae9c Mon Sep 17 00:00:00 2001 From: Chad Woolley Date: Tue, 3 Dec 2024 01:25:56 +0530 Subject: [PATCH 6/6] Fix logic for PAT in e2e script --- scripts/remote_development/run-e2e-tests.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/remote_development/run-e2e-tests.sh b/scripts/remote_development/run-e2e-tests.sh index 0877509bfa3fba..d7507272fb8400 100755 --- a/scripts/remote_development/run-e2e-tests.sh +++ b/scripts/remote_development/run-e2e-tests.sh @@ -11,21 +11,21 @@ DEFAULT_PASSWORD='5iveL!fe' -if [[ -z "${GITLAB_QA_ACCESS_TOKEN}" ]]; then - echo - echo "NOTE: GITLAB_QA_ACCESS_TOKEN is not set. This causes the E2E spec to create a personal access token for the logged in user, which results in a slower test run." - echo " This can be skipped by passing the environment variable GITLAB_QA_ACCESS_TOKEN with PAT of the logged in user" - echo -fi - export WEBDRIVER_HEADLESS="${WEBDRIVER_HEADLESS:-0}" export GITLAB_USERNAME="${GITLAB_USERNAME:-root}" export GITLAB_INITIAL_ROOT_PASSWORD="${GITLAB_INITIAL_ROOT_PASSWORD:-${DEFAULT_PASSWORD}}" -export GITLAB_QA_ACCESS_TOKEN="${PERSONAL_ACCESS_TOKEN:-${TOKEN_VALUE}}" +export GITLAB_QA_ACCESS_TOKEN="${GITLAB_QA_ACCESS_TOKEN:-${PERSONAL_ACCESS_TOKEN}}" export DEVFILE_PROJECT="${DEVFILE_PROJECT:-Gitlab Org / Gitlab Shell}" export AGENT_NAME="${AGENT_NAME:-remotedev}" export TEST_INSTANCE_URL="${TEST_INSTANCE_URL:-http://gdk.test:3000}" +if [[ -z "${GITLAB_QA_ACCESS_TOKEN}" ]]; then + echo + echo "NOTE: GITLAB_QA_ACCESS_TOKEN is not set. This causes the E2E spec to create a personal access token for the logged in user, which results in a slower test run." + echo " This can be skipped by passing the environment variable GITLAB_QA_ACCESS_TOKEN (or PERSONAL_ACCESS_TOKEN) with a valid PAT for the test instance." + echo +fi + echo "WEBDRIVER_HEADLESS: ${WEBDRIVER_HEADLESS}" echo "GITLAB_USERNAME: ${GITLAB_USERNAME}" echo "DEVFILE_PROJECT: ${DEVFILE_PROJECT}" -- GitLab