diff --git a/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml b/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml deleted file mode 100644 index 13ce5174ec1ae1a32c3e0f27e5f5dbf1647a5db9..0000000000000000000000000000000000000000 --- a/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -migration_job_name: BackfillWorkspaceAgentkStates -description: Creates desired_config for each workspace and fills it in the workspace_agentk_states table -feature_category: workspaces -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/199102 -milestone: '18.4' -queued_migration_version: 20250715102036 -finalized_by: # [TBD] version of the migration that finalized this BBM diff --git a/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb b/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb deleted file mode 100644 index e084cb5972e6e683663f185f8da1748bff34be9e..0000000000000000000000000000000000000000 --- a/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -class QueueBackfillWorkspaceAgentkStates < Gitlab::Database::Migration[2.3] - milestone '18.4' - - disable_ddl_transaction! - restrict_gitlab_migration gitlab_schema: :gitlab_main_org - - MIGRATION = "BackfillWorkspaceAgentkStates" - BATCH_SIZE = 1000 - SUB_BATCH_SIZE = 100 - DELAY_INTERVAL = 2.minutes - - # @return [Void] - def up - queue_batched_background_migration( - MIGRATION, - :workspaces, - :id, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE, - job_interval: DELAY_INTERVAL - ) - nil - end - - # @return [Void] - def down - delete_batched_background_migration(MIGRATION, :workspaces, :id, []) - nil - end -end diff --git a/db/schema_migrations/20250715102036 b/db/schema_migrations/20250715102036 deleted file mode 100644 index cceafa1f6bef18fcccbcb7d641162ced5c49dbbb..0000000000000000000000000000000000000000 --- a/db/schema_migrations/20250715102036 +++ /dev/null @@ -1 +0,0 @@ -20b6dad3f68044f89d6e7281a3400fd92a594c2da92b04a7a37df92697c29a32 \ No newline at end of file diff --git a/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb b/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb deleted file mode 100644 index d6f694fa2f170e4f21a09c573d081c56ab65ed3c..0000000000000000000000000000000000000000 --- a/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module EE - module Gitlab - module BackgroundMigration - module BackfillWorkspaceAgentkStates - extend ::Gitlab::Utils::Override - - override :perform - - # rubocop:disable Metrics/MethodLength -- this is a little over the limit, I'll extract some parts to a function if this becomes larger - # @return [Void] - def perform - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspace.reset_column_information - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgent.reset_column_information - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.reset_column_information - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentConfig.reset_column_information - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmAgent.reset_column_information - - # rubocop:disable Metrics/BlockLength -- this is a little over the limit, I'll extract some parts to a function if this becomes larger - each_sub_batch do |sub_batch| - sub_batch.each do |record| - ::Gitlab::BackgroundMigration::RemoteDevelopment::BmCreateDesiredConfig - .create_and_save(workspace_id: record.id) - rescue StandardError => e - message = "Migration failed for this workspace. This workspace will be orphaned, cluster " \ - "administrators are advised to clean up the orphan workspaces." - ::Gitlab::BackgroundMigration::Logger.warn( - message: message, - workspace_id: record.id, - error_message: e.message, - backtrace: e.backtrace&.first(20) - ) - - workspace = ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspace.find(record.id) - workspace.actual_state = "Terminated" - workspace.desired_state = "Terminated" - workspace.save! - - ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.upsert( - { - workspace_id: record.id, - project_id: record.project_id, - desired_config: [{ - message: message, - error_message: e.message, - backtrace: e.backtrace&.first(20) - }] - }, - unique_by: :workspace_id - ) - # rubocop:enable Metrics/BlockLength - end - # rubocop:enable Metrics/MethodLength - end - nil - end - end - end - end -end diff --git a/ee/spec/fixtures/remote_development/background_migration/desired_config.json b/ee/spec/fixtures/remote_development/background_migration/desired_config.json deleted file mode 100644 index 50dac314bb4edc18c9b08a352081a2373db4f7d1..0000000000000000000000000000000000000000 --- a/ee/spec/fixtures/remote_development/background_migration/desired_config.json +++ /dev/null @@ -1,563 +0,0 @@ -{ - "desired_config_array": [ - { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "annotations": { - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1", - "cli-utils.sigs.k8s.io/inventory-id": "$WORKSPACE_NAME-workspace-inventory" - }, - "name": "$WORKSPACE_NAME-workspace-inventory", - "namespace": "$WORKSPACE_NAMESPACE" - } - }, - { - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "creationTimestamp": null, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME", - "namespace": "$WORKSPACE_NAMESPACE" - }, - "spec": { - "replicas": $REPLICA, - "selector": { - "matchLabels": { - "agent.gitlab.com/id": "1" - } - }, - "strategy": { - "type": "Recreate" - }, - "template": { - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "creationTimestamp": null, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME", - "namespace": "$WORKSPACE_NAMESPACE" - }, - "spec": { - "containers": [ - { - "args": [ - "tail -f /dev/null\n" - ], - "command": [ - "/bin/sh", - "-c" - ], - "env": [ - { - "name": "GL_TOOLS_DIR", - "value": "/projects/.gl-tools" - }, - { - "name": "GL_EDITOR_LOG_LEVEL", - "value": "info" - }, - { - "name": "GL_EDITOR_PORT", - "value": "60001" - }, - { - "name": "GL_SSH_PORT", - "value": "60022" - }, - { - "name": "GL_EDITOR_ENABLE_MARKETPLACE", - "value": "false" - }, - { - "name": "PROJECTS_ROOT", - "value": "/projects" - }, - { - "name": "PROJECT_SOURCE", - "value": "/projects" - } - ], - "envFrom": [ - { - "secretRef": { - "name": "$WORKSPACE_NAME-env-var" - } - } - ], - "image": "registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-docs/ubuntu:22.04", - "imagePullPolicy": "Always", - "lifecycle": { - "postStart": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "#!/bin/sh\n\nmkdir -p \"${GL_WORKSPACE_LOGS_DIR}\"\nln -sf \"${GL_WORKSPACE_LOGS_DIR}\" /tmp\n\"/workspace-scripts/gl-run-poststart-commands.sh\" 1>>\"${GL_WORKSPACE_LOGS_DIR}/poststart-stdout.log\" 2>>\"${GL_WORKSPACE_LOGS_DIR}/poststart-stderr.log\"\n" - ] - } - } - }, - "name": "tooling-container", - "ports": [ - { - "containerPort": 60001, - "name": "editor-server", - "protocol": "TCP" - }, - { - "containerPort": 60022, - "name": "ssh-server", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "allowPrivilegeEscalation": false, - "privileged": false, - "runAsNonRoot": true, - "runAsUser": 5001 - }, - "volumeMounts": [ - { - "mountPath": "/projects", - "name": "gl-workspace-data" - }, - { - "mountPath": "/.workspace-data/variables/file", - "name": "gl-workspace-variables" - }, - { - "mountPath": "/workspace-scripts", - "name": "gl-workspace-scripts" - } - ] - } - ], - "initContainers": [ - { - "args": [ - "echo \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Cloning project if necessary...\"\n\nif [ -f \"/projects/.gl_project_cloning_successful\" ]\nthen\n echo \"$(date -Iseconds): Project cloning was already successful\"\n exit 0\nfi\n\nif [ -d \"/projects/gitlab-shell\" ]\nthen\n echo \"$(date -Iseconds): Removing unsuccessfully cloned project directory\"\n rm -rf \"/projects/gitlab-shell\"\nfi\n\necho \"$(date -Iseconds): Cloning project\"\ngit clone --branch \"main\" \"http://gdk.test:3000/gitlab-org/gitlab-shell.git\" \"/projects/gitlab-shell\"\nexit_code=$?\n\nif [ \"${exit_code}\" -eq 0 ]\nthen\n echo \"$(date -Iseconds): Project cloning successful\"\n touch \"/projects/.gl_project_cloning_successful\"\n echo \"$(date -Iseconds): Updated file to indicate successful project cloning\"\nelse\n echo \"$(date -Iseconds): Project cloning failed with exit code: ${exit_code}\" >&2\nfi\n\necho \"$(date -Iseconds): Finished cloning project if necessary.\"\nexit \"${exit_code}\"\n" - ], - "command": [ - "/bin/sh", - "-c" - ], - "env": [ - { - "name": "PROJECTS_ROOT", - "value": "/projects" - }, - { - "name": "PROJECT_SOURCE", - "value": "/projects" - } - ], - "envFrom": [ - { - "secretRef": { - "name": "$WORKSPACE_NAME-env-var" - } - } - ], - "image": "alpine/git:2.45.2", - "imagePullPolicy": "Always", - "name": "gl-project-cloner-gl-project-cloner-command-1", - "resources": { - "limits": { - "cpu": "500m", - "memory": "1000Mi" - }, - "requests": { - "cpu": "100m", - "memory": "500Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "privileged": false, - "runAsNonRoot": true, - "runAsUser": 5001 - }, - "volumeMounts": [ - { - "mountPath": "/projects", - "name": "gl-workspace-data" - }, - { - "mountPath": "/.workspace-data/variables/file", - "name": "gl-workspace-variables" - } - ] - }, - { - "env": [ - { - "name": "GL_TOOLS_DIR", - "value": "/projects/.gl-tools" - }, - { - "name": "PROJECTS_ROOT", - "value": "/projects" - }, - { - "name": "PROJECT_SOURCE", - "value": "/projects" - } - ], - "envFrom": [ - { - "secretRef": { - "name": "$WORKSPACE_NAME-env-var" - } - } - ], - "image": "registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools:15.0.0", - "imagePullPolicy": "Always", - "name": "gl-tools-injector-gl-tools-injector-command-2", - "resources": { - "limits": { - "cpu": "500m", - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "256Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "privileged": false, - "runAsNonRoot": true, - "runAsUser": 5001 - }, - "volumeMounts": [ - { - "mountPath": "/projects", - "name": "gl-workspace-data" - }, - { - "mountPath": "/.workspace-data/variables/file", - "name": "gl-workspace-variables" - } - ] - } - ], - "securityContext": { - "fsGroup": 0, - "fsGroupChangePolicy": "OnRootMismatch", - "runAsNonRoot": true, - "runAsUser": 5001 - }, - "serviceAccountName": "$WORKSPACE_NAME", - "volumes": [ - { - "name": "gl-workspace-data", - "persistentVolumeClaim": { - "claimName": "$WORKSPACE_NAME-gl-workspace-data" - } - }, - { - "name": "gl-workspace-variables", - "projected": { - "defaultMode": 508, - "sources": [ - { - "secret": { - "name": "$WORKSPACE_NAME-file" - } - } - ] - } - }, - { - "name": "gl-workspace-scripts", - "projected": { - "defaultMode": 365, - "sources": [ - { - "configMap": { - "name": "$WORKSPACE_NAME-scripts-configmap" - } - } - ] - } - } - ] - } - } - }, - "status": {} - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "creationTimestamp": null, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME", - "namespace": "$WORKSPACE_NAMESPACE" - }, - "spec": { - "ports": [ - { - "name": "editor-server", - "port": 60001, - "targetPort": 60001 - }, - { - "name": "ssh-server", - "port": 60022, - "targetPort": 60022 - } - ], - "selector": { - "agent.gitlab.com/id": "1" - } - }, - "status": { - "loadBalancer": {} - } - }, - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "creationTimestamp": null, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME-gl-workspace-data", - "namespace": "$WORKSPACE_NAMESPACE" - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "50Gi" - } - } - }, - "status": {} - }, - { - "apiVersion": "v1", - "automountServiceAccountToken": false, - "imagePullSecrets": [], - "kind": "ServiceAccount", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME", - "namespace": "$WORKSPACE_NAMESPACE" - } - }, - { - "apiVersion": "networking.k8s.io/v1", - "kind": "NetworkPolicy", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME", - "namespace": "$WORKSPACE_NAMESPACE" - }, - "spec": { - "egress": [ - { - "ports": [ - { - "port": 53, - "protocol": "TCP" - }, - { - "port": 53, - "protocol": "UDP" - } - ], - "to": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "kube-system" - } - } - } - ] - }, - { - "to": [ - { - "ipBlock": { - "cidr": "0.0.0.0/0", - "except": [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16" - ] - } - } - ] - } - ], - "ingress": [ - { - "from": [ - { - "namespaceSelector": { - "matchLabels": { - "kubernetes.io/metadata.name": "gitlab-workspaces" - } - }, - "podSelector": { - "matchLabels": { - "app.kubernetes.io/name": "gitlab-workspaces-proxy" - } - } - } - ] - } - ], - "podSelector": {}, - "policyTypes": [ - "Ingress", - "Egress" - ] - } - }, - { - "apiVersion": "v1", - "data": { - "gl-init-tools-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running ${GL_TOOLS_DIR}/init_tools.sh with output written to ${GL_WORKSPACE_LOGS_DIR}/init-tools.log...\"\n\"${GL_TOOLS_DIR}/init_tools.sh\" >> \"${GL_WORKSPACE_LOGS_DIR}/init-tools.log\" 2>&1 &\necho \"$(date -Iseconds): Finished running ${GL_TOOLS_DIR}/init_tools.sh.\"\n", - "gl-run-poststart-commands.sh": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-start-sshd-command...\"\n/workspace-scripts/gl-start-sshd-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-start-sshd-command.\"\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-init-tools-command...\"\n/workspace-scripts/gl-init-tools-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-init-tools-command.\"\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-sleep-until-container-is-running-command...\"\n/workspace-scripts/gl-sleep-until-container-is-running-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-sleep-until-container-is-running-command.\"\n", - "gl-sleep-until-container-is-running-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Sleeping until workspace is running...\"\ntime_to_sleep=5\nstatus_file=\"/.workspace-data/variables/file/gl_workspace_reconciled_actual_state.txt\"\nwhile [ \"$(cat ${status_file})\" != \"Running\" ]; do\n echo \"$(date -Iseconds): Workspace state is '$(cat ${status_file})' from status file '${status_file}'. Blocking remaining postStart events execution for ${time_to_sleep} seconds until state is 'Running'...\"\n sleep ${time_to_sleep}\ndone\necho \"$(date -Iseconds): Workspace state is now 'Running', continuing postStart hook execution.\"\necho \"$(date -Iseconds): Finished sleeping until workspace is running.\"\n", - "gl-start-sshd-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Starting sshd if it is found...\"\nsshd_path=$(which sshd)\nif [ -x \"${sshd_path}\" ]; then\n echo \"$(date -Iseconds): Starting ${sshd_path} on port ${GL_SSH_PORT} with output written to ${GL_WORKSPACE_LOGS_DIR}/start-sshd.log\"\n \"${sshd_path}\" -D -p \"${GL_SSH_PORT}\" >> \"${GL_WORKSPACE_LOGS_DIR}/start-sshd.log\" 2>&1 &\nelse\n echo \"$(date -Iseconds): 'sshd' not found in path. Not starting SSH server.\" >&2\nfi\necho \"$(date -Iseconds): Finished starting sshd if it is found.\"\n" - }, - "kind": "ConfigMap", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/include-in-partial-reconciliation": "true", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME-scripts-configmap", - "namespace": "$WORKSPACE_NAMESPACE" - } - }, - { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "annotations": { - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1", - "cli-utils.sigs.k8s.io/inventory-id": "$WORKSPACE_NAME-secrets-inventory" - }, - "name": "$WORKSPACE_NAME-secrets-inventory", - "namespace": "$WORKSPACE_NAMESPACE" - } - }, - { - "apiVersion": "v1", - "data": {}, - "kind": "Secret", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-secrets-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME-env-var", - "namespace": "$WORKSPACE_NAMESPACE" - } - }, - { - "apiVersion": "v1", - "data": {}, - "kind": "Secret", - "metadata": { - "annotations": { - "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-secrets-inventory", - "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", - "workspaces.gitlab.com/id": "$WORKSPACE_ID", - "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - }, - "labels": { - "agent.gitlab.com/id": "1" - }, - "name": "$WORKSPACE_NAME-file", - "namespace": "$WORKSPACE_NAMESPACE" - } - } - ] -} diff --git a/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml b/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml deleted file mode 100644 index 446a5af4f369c0717ef9c4ceb7c97843948e1940..0000000000000000000000000000000000000000 --- a/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml +++ /dev/null @@ -1,157 +0,0 @@ - components: - - attributes: - gl/inject-editor: true - container: - dedicatedPod: false - image: registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-docs/ubuntu:22.04 - mountSources: true - env: - - name: GL_TOOLS_DIR - value: "/projects/.gl-tools" - - name: GL_EDITOR_LOG_LEVEL - value: info - - name: GL_EDITOR_PORT - value: '60001' - - name: GL_SSH_PORT - value: '60022' - - name: GL_EDITOR_ENABLE_MARKETPLACE - value: 'false' - endpoints: - - name: editor-server - targetPort: 60001 - exposure: public - secure: true - protocol: https - - name: ssh-server - targetPort: 60022 - exposure: internal - secure: true - command: - - "/bin/sh" - - "-c" - args: - - 'tail -f /dev/null - - ' - volumeMounts: - - name: gl-workspace-data - path: "/projects" - name: tooling-container - - name: gl-tools-injector - container: - image: registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools:15.0.0 - env: - - name: GL_TOOLS_DIR - value: "/projects/.gl-tools" - memoryLimit: 512Mi - memoryRequest: 256Mi - cpuLimit: 500m - cpuRequest: 100m - volumeMounts: - - name: gl-workspace-data - path: "/projects" - - name: gl-project-cloner - container: - image: alpine/git:2.45.2 - args: - - | - echo "$(date -Iseconds): ----------------------------------------" - echo "$(date -Iseconds): Cloning project if necessary..." - - if [ -f "/projects/.gl_project_cloning_successful" ] - then - echo "$(date -Iseconds): Project cloning was already successful" - exit 0 - fi - - if [ -d "/projects/gitlab-shell" ] - then - echo "$(date -Iseconds): Removing unsuccessfully cloned project directory" - rm -rf "/projects/gitlab-shell" - fi - - echo "$(date -Iseconds): Cloning project" - git clone --branch "main" "http://gdk.test:3000/gitlab-org/gitlab-shell.git" "/projects/gitlab-shell" - exit_code=$? - - if [ "${exit_code}" -eq 0 ] - then - echo "$(date -Iseconds): Project cloning successful" - touch "/projects/.gl_project_cloning_successful" - echo "$(date -Iseconds): Updated file to indicate successful project cloning" - else - echo "$(date -Iseconds): Project cloning failed with exit code: ${exit_code}" >&2 - fi - - echo "$(date -Iseconds): Finished cloning project if necessary." - exit "${exit_code}" - command: - - "/bin/sh" - - "-c" - memoryLimit: 1000Mi - memoryRequest: 500Mi - cpuLimit: 500m - cpuRequest: 100m - volumeMounts: - - name: gl-workspace-data - path: "/projects" - - name: gl-workspace-data - volume: - size: 50Gi - metadata: {} - schemaVersion: 2.2.0 - commands: - - id: gl-tools-injector-command - apply: - component: gl-tools-injector - - id: gl-start-sshd-command - exec: - commandLine: | - #!/bin/sh - echo "$(date -Iseconds): ----------------------------------------" - echo "$(date -Iseconds): Starting sshd if it is found..." - sshd_path=$(which sshd) - if [ -x "${sshd_path}" ]; then - echo "$(date -Iseconds): Starting ${sshd_path} on port ${GL_SSH_PORT} with output written to ${GL_WORKSPACE_LOGS_DIR}/start-sshd.log" - "${sshd_path}" -D -p "${GL_SSH_PORT}" >> "${GL_WORKSPACE_LOGS_DIR}/start-sshd.log" 2>&1 & - else - echo "$(date -Iseconds): 'sshd' not found in path. Not starting SSH server." >&2 - fi - echo "$(date -Iseconds): Finished starting sshd if it is found." - component: tooling-container - - id: gl-init-tools-command - exec: - commandLine: | - #!/bin/sh - echo "$(date -Iseconds): ----------------------------------------" - echo "$(date -Iseconds): Running ${GL_TOOLS_DIR}/init_tools.sh with output written to ${GL_WORKSPACE_LOGS_DIR}/init-tools.log..." - "${GL_TOOLS_DIR}/init_tools.sh" >> "${GL_WORKSPACE_LOGS_DIR}/init-tools.log" 2>&1 & - echo "$(date -Iseconds): Finished running ${GL_TOOLS_DIR}/init_tools.sh." - component: tooling-container - - id: gl-sleep-until-container-is-running-command - exec: - commandLine: | - #!/bin/sh - echo "$(date -Iseconds): ----------------------------------------" - echo "$(date -Iseconds): Sleeping until workspace is running..." - time_to_sleep=5 - status_file="/.workspace-data/variables/file/gl_workspace_reconciled_actual_state.txt" - while [ "$(cat ${status_file})" != "Running" ]; do - echo "$(date -Iseconds): Workspace state is '$(cat ${status_file})' from status file '${status_file}'. Blocking remaining postStart events execution for ${time_to_sleep} seconds until state is 'Running'..." - sleep ${time_to_sleep} - done - echo "$(date -Iseconds): Workspace state is now 'Running', continuing postStart hook execution." - echo "$(date -Iseconds): Finished sleeping until workspace is running." - component: tooling-container - - id: gl-project-cloner-command - apply: - component: gl-project-cloner - events: - preStart: - - gl-tools-injector-command - - gl-project-cloner-command - postStart: - - gl-start-sshd-command - - gl-init-tools-command - - gl-sleep-until-container-is-running-command - variables: {} diff --git a/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb deleted file mode 100644 index 675cbfff1c74bd99534eac08054915e5033eff89..0000000000000000000000000000000000000000 --- a/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb +++ /dev/null @@ -1,248 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -RSpec.describe Gitlab::BackgroundMigration::BackfillWorkspaceAgentkStates, feature_category: :workspaces do - let(:organization) { table(:organizations).create!(name: "test-org", path: "default") } - let(:processed_devfile) do - read_fixture_file("processed_devfile.yaml") - end - - let!(:workspace_1) do - table(:workspaces).create!( - user_id: user.id, - project_id: project.id, - cluster_agent_id: agent.id, - desired_state_updated_at: Time.now.utc, - actual_state_updated_at: Time.now.utc, - responded_to_agent_at: Time.now.utc, - name: "workspace-1", - namespace: "workspace_1_namespace", - desired_state: "Terminated", - actual_state: "Terminated", - project_ref: "devfile-ref", - devfile_path: "devfile-path", - devfile: devfile, - processed_devfile: processed_devfile, - url: "workspace-url", - deployment_resource_version: "v1", - personal_access_token_id: personal_access_token.id, - workspaces_agent_config_version: agent_config_version.id, - desired_config_generator_version: 3 - ) - end - - let!(:workspace_2) do - table(:workspaces).create!( - user_id: user.id, - project_id: project.id, - cluster_agent_id: agent.id, - desired_state_updated_at: Time.now.utc, - actual_state_updated_at: Time.now.utc, - responded_to_agent_at: Time.now.utc, - name: "workspace-2", - namespace: "workspace_2_namespace", - desired_state: "Running", - actual_state: "Running", - project_ref: "devfile-ref", - devfile_path: "devfile-path", - devfile: devfile, - processed_devfile: processed_devfile, - url: "workspace-url", - deployment_resource_version: "v1", - personal_access_token_id: personal_access_token.id, - workspaces_agent_config_version: agent_config_version.id, - desired_config_generator_version: 3 - ) - end - - let(:migration) do - described_class.new( - start_id: workspace_1.id, - end_id: workspace_2.id, - batch_table: :workspaces, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ApplicationRecord.connection - ) - end - - let(:expected_desired_config_workspace_1) do - json_str = read_fixture_file("desired_config.json") - json_str.gsub!("$WORKSPACE_ID", workspace_1.id.to_s) - json_str.gsub!("$WORKSPACE_NAMESPACE", workspace_1.namespace) - json_str.gsub!("$WORKSPACE_NAME", workspace_1.name) - json_str.gsub!("$REPLICA", "0") - ::Gitlab::Json.parse(json_str) - end - - let(:expected_desired_config_workspace_2) do - json_str = read_fixture_file("desired_config.json") - json_str.gsub!("$WORKSPACE_ID", workspace_2.id.to_s) - json_str.gsub!("$WORKSPACE_NAMESPACE", workspace_2.namespace) - json_str.gsub!("$WORKSPACE_NAME", workspace_2.name) - json_str.gsub!("$REPLICA", "1") - ::Gitlab::Json.parse(json_str) - end - - let(:user) do - table(:users).create!( - name: "test", - email: "test@example.com", - projects_limit: 5, - organization_id: organization.id - ) - end - - let(:namespace) { table(:namespaces).create!(name: "name", path: "path", organization_id: organization.id) } - let(:project) do - table(:projects).create!( - namespace_id: namespace.id, - project_namespace_id: namespace.id, - organization_id: organization.id - ) - end - - let!(:personal_access_token) do - table(:personal_access_tokens).create!( - user_id: user.id, - name: "token_name", - expires_at: Time.now.utc, - organization_id: organization.id - ) - end - - let(:agent) do - table(:cluster_agents).create!( - id: 1, - name: "Agent-1", - project_id: project.id - ) - end - - let!(:agent_config) do - table(:workspaces_agent_configs).create!( - cluster_agent_id: agent.id, - enabled: true, - dns_zone: "test.workspace.me", - project_id: project.id - ) - end - - let!(:agent_config_version) do - table(:workspaces_agent_config_versions).create!( - project_id: project.id, - item_id: agent_config.id, - item_type: "Gitlab::BackgroundMigration::RemoteDevelopment::BMWorkspacesAgentConfig", - event: "create" - ) - end - - let(:devfile) do - <<~YAML - schemaVersion: 2.2.0 - components: - - name: tooling-container - attributes: - gl/inject-editor: true - container: - image: registry.gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/debian-bullseye-ruby-3.2-node-18.12:rubygems-3.4-git-2.33-lfs-2.9-yarn-1.22-graphicsmagick-1.3.36-gitlab-workspaces - env: - - name: KEY - value: VALUE - endpoints: - - name: http-3000 - targetPort: 3000 - YAML - end - - # @param [String] filename - # @return [String] - def read_fixture_file(filename) - File.read(Rails.root.join("ee/spec/fixtures/remote_development/background_migration", filename).to_s) - end - - context "when desired_config is valid" do - it "creates a record workspace_agentk_states table for each workspace", :unlimited_max_formatted_output_length do - expect { migration.perform } - .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } - .by(2) - - workspace_1_agentk_state_post_migration = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) - workspace_2_agentk_state_post_migration = table(:workspace_agentk_states).find_by(workspace_id: workspace_2.id) - - expect(workspace_1_agentk_state_post_migration.project_id).to eq(workspace_1.project_id) - expect(workspace_2_agentk_state_post_migration.project_id).to eq(workspace_2.project_id) - - expect(workspace_1_agentk_state_post_migration.desired_config).to eq(expected_desired_config_workspace_1) - expect(workspace_2_agentk_state_post_migration.desired_config).to eq(expected_desired_config_workspace_2) - end - end - - context "when desired_config is invalid" do - it "creates a record in workspace_agentk_states with failed message and terminates the workspace", - :unlimited_max_formatted_output_length do - error = Devfile::CliError.new( - "quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'" - ) - allow(error).to receive(:backtrace).and_return([ - "/app/lib/some_file.rb:123:in `method_name'", - "/app/lib/another_file.rb:456:in `another_method'" - ]) - - allow(Gitlab::BackgroundMigration::RemoteDevelopment::WorkspaceOperations::Create::DesiredConfig::BmMain) - .to receive(:main).and_raise(error) - - expect { migration.perform } - .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } - .by(2) - - workspace_1_updated = table(:workspaces).find(workspace_1.id) - workspace_2_updated = table(:workspaces).find(workspace_2.id) - - expect(workspace_1_updated.actual_state).to eq("Terminated") - expect(workspace_1_updated.desired_state).to eq("Terminated") - expect(workspace_2_updated.actual_state).to eq("Terminated") - expect(workspace_2_updated.desired_state).to eq("Terminated") - - saved_records = Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.all - saved_records.each do |record| - expect(record.desired_config).to include( - { - "error_message" => "quantities must match the regular expression " \ - "'^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'", - "message" => "Migration failed for this workspace. This workspace will be orphaned, cluster " \ - "administrators are advised to clean up the orphan workspaces.", - "backtrace" => [ - "/app/lib/some_file.rb:123:in `method_name'", - "/app/lib/another_file.rb:456:in `another_method'" - ] - } - ) - end - end - end - - context "when config already exists for a workspace" do - it "skips updating the config for that workspace", :unlimited_max_formatted_output_length do - workspace_2_agentk_state = table(:workspace_agentk_states).create!( - desired_config: { "some_key" => "some_value" }, - workspace_id: workspace_2.id, - project_id: workspace_2.project_id - ) - - expect(workspace_2_agentk_state).to be_persisted - - workspace_1_agentk_state = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) - expect(workspace_1_agentk_state).to be_nil - - expect { migration.perform } - .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } - .by(1) - - workspace_1_agentk_state = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) - expect(workspace_1_agentk_state).not_to be_nil - end - end -end diff --git a/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb b/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb deleted file mode 100644 index 9cbad4fd56a77210710e3960b4e77dc800291f3a..0000000000000000000000000000000000000000 --- a/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - class BackfillWorkspaceAgentkStates < BatchedMigrationJob - operation_name :backfill_workspace_agentk_states - feature_category :workspaces - - # @return [Void] - def perform; end - end - end -end - -Gitlab::BackgroundMigration::BackfillWorkspaceAgentkStates.prepend_mod diff --git a/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb b/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb index 267f0cc37c5df5fe23b0e91a01c169243592213d..e9001f87e952c1d4629ea343ccd0867a96a70126 100644 --- a/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb +++ b/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb @@ -28,13 +28,12 @@ def self.create_and_save(workspace_id:, dry_run: false) ) end - # rubocop:disable Metrics/MethodLength -- need it big # @param [BackgroundMigration::RemoteDevelopment::Models::BMWorkspace] workspace # @param [BackgroundMigration::RemoteDevelopment::WorkspaceOperations::BMDesiredConfig] desired_config # @param [Gitlab::BackgroundMigration::Logger] logger # @param [Boolean] dry_run # @return [Void] - def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, logger:, dry_run:) + def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, logger:, dry_run:) # rubocop:disable Metrics/MethodLength -- need it big if dry_run puts "For workspace_id #{workspace.id}" puts "Valid desired_config? #{desired_config.valid?}" @@ -55,7 +54,6 @@ def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, end if dry_run - # noinspection RubyArgCount -- RubyMine does not recognize the fields workspace_agentk_state = RemoteDevelopment::Models::BmWorkspaceAgentkState.new( workspace_id: workspace.id, project_id: workspace.project_id, @@ -67,21 +65,17 @@ def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, puts message end else - RemoteDevelopment::Models::BmWorkspaceAgentkState.upsert( - { - workspace_id: workspace.id, - project_id: workspace.project_id, - desired_config: desired_config - }, - unique_by: :workspace_id + RemoteDevelopment::Models::BmWorkspaceAgentkState.create!( + workspace_id: workspace.id, + project_id: workspace.project_id, + desired_config: desired_config ) end end - # rubocop:enable Metrics/MethodLength # @return [Gitlab::BackgroundMigration::Logger] def self.logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build + @logger ||= Gitlab::BackgroundMigration::Logger.build end end end diff --git a/lib/gitlab/background_migration/remote_development/models/bm_agent.rb b/lib/gitlab/background_migration/remote_development/models/bm_agent.rb deleted file mode 100644 index 279bf3d5ab8b4ca408dc6a90c1e5dd4ad39fdccb..0000000000000000000000000000000000000000 --- a/lib/gitlab/background_migration/remote_development/models/bm_agent.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module RemoteDevelopment - module Models - # rubocop:disable Migration/BatchedMigrationBaseClass -- This is not a migration file class so we do not need to inherit from BatchedMigrationJob - class BmAgent < ::Gitlab::Database::Migration[2.3]::MigrationRecord - self.table_name = 'cluster_agents' - - has_one :unversioned_latest_workspaces_agent_config, - class_name: 'RemoteDevelopment::Models::BmWorkspaceAgentConfig', - inverse_of: :agent, - foreign_key: :cluster_agent_id - - has_many :workspaces, - class_name: 'RemoteDevelopment::Models::BmWorkspace', - inverse_of: :agent, - foreign_key: :cluster_agent_id - end - - # rubocop:enable Migration/BatchedMigrationBaseClass - end - end - end -end diff --git a/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb b/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb index a3a4d6871b31a412d97ce528f0ec2118aa9bd20f..35f8c33ddfb1a7cb0004a27f49607323a7a159c8 100644 --- a/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb +++ b/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb @@ -10,7 +10,7 @@ class BmWorkspace < ::Gitlab::Database::Migration[2.3]::MigrationRecord self.table_name = 'workspaces' - belongs_to :agent, class_name: "BmAgent", foreign_key: "cluster_agent_id", inverse_of: :workspaces + belongs_to :agent, class_name: "Clusters::Agent", foreign_key: "cluster_agent_id", inverse_of: :workspaces # @return [Boolean] def desired_state_running? diff --git a/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb b/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb index 3dec2d9e9d960b0977d36c5aa8ee86b9ad18655e..2cdf5252bd500707ef7687dbd1f64cf465ea94ce 100644 --- a/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb +++ b/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb @@ -8,10 +8,10 @@ module Models class BmWorkspaceAgentConfig < ::Gitlab::Database::Migration[2.3]::MigrationRecord include WorkspaceOperations::BmStates - self.table_name = 'workspaces_agent_configs' + self.table_name = 'workspace_agent_configs' belongs_to :agent, - class_name: 'BmAgent', foreign_key: 'cluster_agent_id', + class_name: 'Clusters::Agent', foreign_key: 'cluster_agent_id', inverse_of: :unversioned_latest_workspaces_agent_config end # rubocop:enable Migration/BatchedMigrationBaseClass diff --git a/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb b/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb deleted file mode 100644 index 49fc46d1815c8fa64129acd8553e2d5cc4fdb070..0000000000000000000000000000000000000000 --- a/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe QueueBackfillWorkspaceAgentkStates, migration: :gitlab_main_org, feature_category: :workspaces do - let!(:batched_migration) { described_class::MIGRATION } - - it 'schedules a new batched migration' do - reversible_migration do |migration| - migration.before -> { - expect(batched_migration).not_to have_scheduled_batched_migration - } - - migration.after -> { - expect(batched_migration).to have_scheduled_batched_migration( - gitlab_schema: :gitlab_main_org, - table_name: :workspaces, - column_name: :id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - sub_batch_size: described_class::SUB_BATCH_SIZE - ) - } - end - end -end