diff --git a/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml b/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml new file mode 100644 index 0000000000000000000000000000000000000000..13ce5174ec1ae1a32c3e0f27e5f5dbf1647a5db9 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_workspace_agentk_states.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillWorkspaceAgentkStates +description: Creates desired_config for each workspace and fills it in the workspace_agentk_states table +feature_category: workspaces +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/199102 +milestone: '18.4' +queued_migration_version: 20250715102036 +finalized_by: # [TBD] version of the migration that finalized this BBM diff --git a/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb b/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..e084cb5972e6e683663f185f8da1748bff34be9e --- /dev/null +++ b/db/post_migrate/20250715102036_queue_backfill_workspace_agentk_states.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class QueueBackfillWorkspaceAgentkStates < Gitlab::Database::Migration[2.3] + milestone '18.4' + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main_org + + MIGRATION = "BackfillWorkspaceAgentkStates" + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + + # @return [Void] + def up + queue_batched_background_migration( + MIGRATION, + :workspaces, + :id, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + job_interval: DELAY_INTERVAL + ) + nil + end + + # @return [Void] + def down + delete_batched_background_migration(MIGRATION, :workspaces, :id, []) + nil + end +end diff --git a/db/schema_migrations/20250715102036 b/db/schema_migrations/20250715102036 new file mode 100644 index 0000000000000000000000000000000000000000..cceafa1f6bef18fcccbcb7d641162ced5c49dbbb --- /dev/null +++ b/db/schema_migrations/20250715102036 @@ -0,0 +1 @@ +20b6dad3f68044f89d6e7281a3400fd92a594c2da92b04a7a37df92697c29a32 \ No newline at end of file diff --git a/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb b/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..d6f694fa2f170e4f21a09c573d081c56ab65ed3c --- /dev/null +++ b/ee/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module EE + module Gitlab + module BackgroundMigration + module BackfillWorkspaceAgentkStates + extend ::Gitlab::Utils::Override + + override :perform + + # rubocop:disable Metrics/MethodLength -- this is a little over the limit, I'll extract some parts to a function if this becomes larger + # @return [Void] + def perform + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspace.reset_column_information + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgent.reset_column_information + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.reset_column_information + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentConfig.reset_column_information + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmAgent.reset_column_information + + # rubocop:disable Metrics/BlockLength -- this is a little over the limit, I'll extract some parts to a function if this becomes larger + each_sub_batch do |sub_batch| + sub_batch.each do |record| + ::Gitlab::BackgroundMigration::RemoteDevelopment::BmCreateDesiredConfig + .create_and_save(workspace_id: record.id) + rescue StandardError => e + message = "Migration failed for this workspace. This workspace will be orphaned, cluster " \ + "administrators are advised to clean up the orphan workspaces." + ::Gitlab::BackgroundMigration::Logger.warn( + message: message, + workspace_id: record.id, + error_message: e.message, + backtrace: e.backtrace&.first(20) + ) + + workspace = ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspace.find(record.id) + workspace.actual_state = "Terminated" + workspace.desired_state = "Terminated" + workspace.save! + + ::Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.upsert( + { + workspace_id: record.id, + project_id: record.project_id, + desired_config: [{ + message: message, + error_message: e.message, + backtrace: e.backtrace&.first(20) + }] + }, + unique_by: :workspace_id + ) + # rubocop:enable Metrics/BlockLength + end + # rubocop:enable Metrics/MethodLength + end + nil + end + end + end + end +end diff --git a/ee/spec/fixtures/remote_development/background_migration/desired_config.json b/ee/spec/fixtures/remote_development/background_migration/desired_config.json new file mode 100644 index 0000000000000000000000000000000000000000..50dac314bb4edc18c9b08a352081a2373db4f7d1 --- /dev/null +++ b/ee/spec/fixtures/remote_development/background_migration/desired_config.json @@ -0,0 +1,563 @@ +{ + "desired_config_array": [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "annotations": { + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1", + "cli-utils.sigs.k8s.io/inventory-id": "$WORKSPACE_NAME-workspace-inventory" + }, + "name": "$WORKSPACE_NAME-workspace-inventory", + "namespace": "$WORKSPACE_NAMESPACE" + } + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "creationTimestamp": null, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME", + "namespace": "$WORKSPACE_NAMESPACE" + }, + "spec": { + "replicas": $REPLICA, + "selector": { + "matchLabels": { + "agent.gitlab.com/id": "1" + } + }, + "strategy": { + "type": "Recreate" + }, + "template": { + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "creationTimestamp": null, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME", + "namespace": "$WORKSPACE_NAMESPACE" + }, + "spec": { + "containers": [ + { + "args": [ + "tail -f /dev/null\n" + ], + "command": [ + "/bin/sh", + "-c" + ], + "env": [ + { + "name": "GL_TOOLS_DIR", + "value": "/projects/.gl-tools" + }, + { + "name": "GL_EDITOR_LOG_LEVEL", + "value": "info" + }, + { + "name": "GL_EDITOR_PORT", + "value": "60001" + }, + { + "name": "GL_SSH_PORT", + "value": "60022" + }, + { + "name": "GL_EDITOR_ENABLE_MARKETPLACE", + "value": "false" + }, + { + "name": "PROJECTS_ROOT", + "value": "/projects" + }, + { + "name": "PROJECT_SOURCE", + "value": "/projects" + } + ], + "envFrom": [ + { + "secretRef": { + "name": "$WORKSPACE_NAME-env-var" + } + } + ], + "image": "registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-docs/ubuntu:22.04", + "imagePullPolicy": "Always", + "lifecycle": { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "#!/bin/sh\n\nmkdir -p \"${GL_WORKSPACE_LOGS_DIR}\"\nln -sf \"${GL_WORKSPACE_LOGS_DIR}\" /tmp\n\"/workspace-scripts/gl-run-poststart-commands.sh\" 1>>\"${GL_WORKSPACE_LOGS_DIR}/poststart-stdout.log\" 2>>\"${GL_WORKSPACE_LOGS_DIR}/poststart-stderr.log\"\n" + ] + } + } + }, + "name": "tooling-container", + "ports": [ + { + "containerPort": 60001, + "name": "editor-server", + "protocol": "TCP" + }, + { + "containerPort": 60022, + "name": "ssh-server", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "allowPrivilegeEscalation": false, + "privileged": false, + "runAsNonRoot": true, + "runAsUser": 5001 + }, + "volumeMounts": [ + { + "mountPath": "/projects", + "name": "gl-workspace-data" + }, + { + "mountPath": "/.workspace-data/variables/file", + "name": "gl-workspace-variables" + }, + { + "mountPath": "/workspace-scripts", + "name": "gl-workspace-scripts" + } + ] + } + ], + "initContainers": [ + { + "args": [ + "echo \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Cloning project if necessary...\"\n\nif [ -f \"/projects/.gl_project_cloning_successful\" ]\nthen\n echo \"$(date -Iseconds): Project cloning was already successful\"\n exit 0\nfi\n\nif [ -d \"/projects/gitlab-shell\" ]\nthen\n echo \"$(date -Iseconds): Removing unsuccessfully cloned project directory\"\n rm -rf \"/projects/gitlab-shell\"\nfi\n\necho \"$(date -Iseconds): Cloning project\"\ngit clone --branch \"main\" \"http://gdk.test:3000/gitlab-org/gitlab-shell.git\" \"/projects/gitlab-shell\"\nexit_code=$?\n\nif [ \"${exit_code}\" -eq 0 ]\nthen\n echo \"$(date -Iseconds): Project cloning successful\"\n touch \"/projects/.gl_project_cloning_successful\"\n echo \"$(date -Iseconds): Updated file to indicate successful project cloning\"\nelse\n echo \"$(date -Iseconds): Project cloning failed with exit code: ${exit_code}\" >&2\nfi\n\necho \"$(date -Iseconds): Finished cloning project if necessary.\"\nexit \"${exit_code}\"\n" + ], + "command": [ + "/bin/sh", + "-c" + ], + "env": [ + { + "name": "PROJECTS_ROOT", + "value": "/projects" + }, + { + "name": "PROJECT_SOURCE", + "value": "/projects" + } + ], + "envFrom": [ + { + "secretRef": { + "name": "$WORKSPACE_NAME-env-var" + } + } + ], + "image": "alpine/git:2.45.2", + "imagePullPolicy": "Always", + "name": "gl-project-cloner-gl-project-cloner-command-1", + "resources": { + "limits": { + "cpu": "500m", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "500Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "privileged": false, + "runAsNonRoot": true, + "runAsUser": 5001 + }, + "volumeMounts": [ + { + "mountPath": "/projects", + "name": "gl-workspace-data" + }, + { + "mountPath": "/.workspace-data/variables/file", + "name": "gl-workspace-variables" + } + ] + }, + { + "env": [ + { + "name": "GL_TOOLS_DIR", + "value": "/projects/.gl-tools" + }, + { + "name": "PROJECTS_ROOT", + "value": "/projects" + }, + { + "name": "PROJECT_SOURCE", + "value": "/projects" + } + ], + "envFrom": [ + { + "secretRef": { + "name": "$WORKSPACE_NAME-env-var" + } + } + ], + "image": "registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools:15.0.0", + "imagePullPolicy": "Always", + "name": "gl-tools-injector-gl-tools-injector-command-2", + "resources": { + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "256Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "privileged": false, + "runAsNonRoot": true, + "runAsUser": 5001 + }, + "volumeMounts": [ + { + "mountPath": "/projects", + "name": "gl-workspace-data" + }, + { + "mountPath": "/.workspace-data/variables/file", + "name": "gl-workspace-variables" + } + ] + } + ], + "securityContext": { + "fsGroup": 0, + "fsGroupChangePolicy": "OnRootMismatch", + "runAsNonRoot": true, + "runAsUser": 5001 + }, + "serviceAccountName": "$WORKSPACE_NAME", + "volumes": [ + { + "name": "gl-workspace-data", + "persistentVolumeClaim": { + "claimName": "$WORKSPACE_NAME-gl-workspace-data" + } + }, + { + "name": "gl-workspace-variables", + "projected": { + "defaultMode": 508, + "sources": [ + { + "secret": { + "name": "$WORKSPACE_NAME-file" + } + } + ] + } + }, + { + "name": "gl-workspace-scripts", + "projected": { + "defaultMode": 365, + "sources": [ + { + "configMap": { + "name": "$WORKSPACE_NAME-scripts-configmap" + } + } + ] + } + } + ] + } + } + }, + "status": {} + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "creationTimestamp": null, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME", + "namespace": "$WORKSPACE_NAMESPACE" + }, + "spec": { + "ports": [ + { + "name": "editor-server", + "port": 60001, + "targetPort": 60001 + }, + { + "name": "ssh-server", + "port": 60022, + "targetPort": 60022 + } + ], + "selector": { + "agent.gitlab.com/id": "1" + } + }, + "status": { + "loadBalancer": {} + } + }, + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "creationTimestamp": null, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME-gl-workspace-data", + "namespace": "$WORKSPACE_NAMESPACE" + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "50Gi" + } + } + }, + "status": {} + }, + { + "apiVersion": "v1", + "automountServiceAccountToken": false, + "imagePullSecrets": [], + "kind": "ServiceAccount", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME", + "namespace": "$WORKSPACE_NAMESPACE" + } + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME", + "namespace": "$WORKSPACE_NAMESPACE" + }, + "spec": { + "egress": [ + { + "ports": [ + { + "port": 53, + "protocol": "TCP" + }, + { + "port": 53, + "protocol": "UDP" + } + ], + "to": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "kube-system" + } + } + } + ] + }, + { + "to": [ + { + "ipBlock": { + "cidr": "0.0.0.0/0", + "except": [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] + } + } + ] + } + ], + "ingress": [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "gitlab-workspaces" + } + }, + "podSelector": { + "matchLabels": { + "app.kubernetes.io/name": "gitlab-workspaces-proxy" + } + } + } + ] + } + ], + "podSelector": {}, + "policyTypes": [ + "Ingress", + "Egress" + ] + } + }, + { + "apiVersion": "v1", + "data": { + "gl-init-tools-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running ${GL_TOOLS_DIR}/init_tools.sh with output written to ${GL_WORKSPACE_LOGS_DIR}/init-tools.log...\"\n\"${GL_TOOLS_DIR}/init_tools.sh\" >> \"${GL_WORKSPACE_LOGS_DIR}/init-tools.log\" 2>&1 &\necho \"$(date -Iseconds): Finished running ${GL_TOOLS_DIR}/init_tools.sh.\"\n", + "gl-run-poststart-commands.sh": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-start-sshd-command...\"\n/workspace-scripts/gl-start-sshd-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-start-sshd-command.\"\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-init-tools-command...\"\n/workspace-scripts/gl-init-tools-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-init-tools-command.\"\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Running /workspace-scripts/gl-sleep-until-container-is-running-command...\"\n/workspace-scripts/gl-sleep-until-container-is-running-command || true\necho \"$(date -Iseconds): Finished running /workspace-scripts/gl-sleep-until-container-is-running-command.\"\n", + "gl-sleep-until-container-is-running-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Sleeping until workspace is running...\"\ntime_to_sleep=5\nstatus_file=\"/.workspace-data/variables/file/gl_workspace_reconciled_actual_state.txt\"\nwhile [ \"$(cat ${status_file})\" != \"Running\" ]; do\n echo \"$(date -Iseconds): Workspace state is '$(cat ${status_file})' from status file '${status_file}'. Blocking remaining postStart events execution for ${time_to_sleep} seconds until state is 'Running'...\"\n sleep ${time_to_sleep}\ndone\necho \"$(date -Iseconds): Workspace state is now 'Running', continuing postStart hook execution.\"\necho \"$(date -Iseconds): Finished sleeping until workspace is running.\"\n", + "gl-start-sshd-command": "#!/bin/sh\necho \"$(date -Iseconds): ----------------------------------------\"\necho \"$(date -Iseconds): Starting sshd if it is found...\"\nsshd_path=$(which sshd)\nif [ -x \"${sshd_path}\" ]; then\n echo \"$(date -Iseconds): Starting ${sshd_path} on port ${GL_SSH_PORT} with output written to ${GL_WORKSPACE_LOGS_DIR}/start-sshd.log\"\n \"${sshd_path}\" -D -p \"${GL_SSH_PORT}\" >> \"${GL_WORKSPACE_LOGS_DIR}/start-sshd.log\" 2>&1 &\nelse\n echo \"$(date -Iseconds): 'sshd' not found in path. Not starting SSH server.\" >&2\nfi\necho \"$(date -Iseconds): Finished starting sshd if it is found.\"\n" + }, + "kind": "ConfigMap", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-workspace-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/include-in-partial-reconciliation": "true", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME-scripts-configmap", + "namespace": "$WORKSPACE_NAMESPACE" + } + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "annotations": { + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1", + "cli-utils.sigs.k8s.io/inventory-id": "$WORKSPACE_NAME-secrets-inventory" + }, + "name": "$WORKSPACE_NAME-secrets-inventory", + "namespace": "$WORKSPACE_NAMESPACE" + } + }, + { + "apiVersion": "v1", + "data": {}, + "kind": "Secret", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-secrets-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME-env-var", + "namespace": "$WORKSPACE_NAMESPACE" + } + }, + { + "apiVersion": "v1", + "data": {}, + "kind": "Secret", + "metadata": { + "annotations": { + "config.k8s.io/owning-inventory": "$WORKSPACE_NAME-secrets-inventory", + "workspaces.gitlab.com/host-template": "{{.port}}-$WORKSPACE_NAME.test.workspace.me", + "workspaces.gitlab.com/id": "$WORKSPACE_ID", + "workspaces.gitlab.com/max-resources-per-workspace-sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "labels": { + "agent.gitlab.com/id": "1" + }, + "name": "$WORKSPACE_NAME-file", + "namespace": "$WORKSPACE_NAMESPACE" + } + } + ] +} diff --git a/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml b/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml new file mode 100644 index 0000000000000000000000000000000000000000..446a5af4f369c0717ef9c4ceb7c97843948e1940 --- /dev/null +++ b/ee/spec/fixtures/remote_development/background_migration/processed_devfile.yaml @@ -0,0 +1,157 @@ + components: + - attributes: + gl/inject-editor: true + container: + dedicatedPod: false + image: registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-docs/ubuntu:22.04 + mountSources: true + env: + - name: GL_TOOLS_DIR + value: "/projects/.gl-tools" + - name: GL_EDITOR_LOG_LEVEL + value: info + - name: GL_EDITOR_PORT + value: '60001' + - name: GL_SSH_PORT + value: '60022' + - name: GL_EDITOR_ENABLE_MARKETPLACE + value: 'false' + endpoints: + - name: editor-server + targetPort: 60001 + exposure: public + secure: true + protocol: https + - name: ssh-server + targetPort: 60022 + exposure: internal + secure: true + command: + - "/bin/sh" + - "-c" + args: + - 'tail -f /dev/null + + ' + volumeMounts: + - name: gl-workspace-data + path: "/projects" + name: tooling-container + - name: gl-tools-injector + container: + image: registry.gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools:15.0.0 + env: + - name: GL_TOOLS_DIR + value: "/projects/.gl-tools" + memoryLimit: 512Mi + memoryRequest: 256Mi + cpuLimit: 500m + cpuRequest: 100m + volumeMounts: + - name: gl-workspace-data + path: "/projects" + - name: gl-project-cloner + container: + image: alpine/git:2.45.2 + args: + - | + echo "$(date -Iseconds): ----------------------------------------" + echo "$(date -Iseconds): Cloning project if necessary..." + + if [ -f "/projects/.gl_project_cloning_successful" ] + then + echo "$(date -Iseconds): Project cloning was already successful" + exit 0 + fi + + if [ -d "/projects/gitlab-shell" ] + then + echo "$(date -Iseconds): Removing unsuccessfully cloned project directory" + rm -rf "/projects/gitlab-shell" + fi + + echo "$(date -Iseconds): Cloning project" + git clone --branch "main" "http://gdk.test:3000/gitlab-org/gitlab-shell.git" "/projects/gitlab-shell" + exit_code=$? + + if [ "${exit_code}" -eq 0 ] + then + echo "$(date -Iseconds): Project cloning successful" + touch "/projects/.gl_project_cloning_successful" + echo "$(date -Iseconds): Updated file to indicate successful project cloning" + else + echo "$(date -Iseconds): Project cloning failed with exit code: ${exit_code}" >&2 + fi + + echo "$(date -Iseconds): Finished cloning project if necessary." + exit "${exit_code}" + command: + - "/bin/sh" + - "-c" + memoryLimit: 1000Mi + memoryRequest: 500Mi + cpuLimit: 500m + cpuRequest: 100m + volumeMounts: + - name: gl-workspace-data + path: "/projects" + - name: gl-workspace-data + volume: + size: 50Gi + metadata: {} + schemaVersion: 2.2.0 + commands: + - id: gl-tools-injector-command + apply: + component: gl-tools-injector + - id: gl-start-sshd-command + exec: + commandLine: | + #!/bin/sh + echo "$(date -Iseconds): ----------------------------------------" + echo "$(date -Iseconds): Starting sshd if it is found..." + sshd_path=$(which sshd) + if [ -x "${sshd_path}" ]; then + echo "$(date -Iseconds): Starting ${sshd_path} on port ${GL_SSH_PORT} with output written to ${GL_WORKSPACE_LOGS_DIR}/start-sshd.log" + "${sshd_path}" -D -p "${GL_SSH_PORT}" >> "${GL_WORKSPACE_LOGS_DIR}/start-sshd.log" 2>&1 & + else + echo "$(date -Iseconds): 'sshd' not found in path. Not starting SSH server." >&2 + fi + echo "$(date -Iseconds): Finished starting sshd if it is found." + component: tooling-container + - id: gl-init-tools-command + exec: + commandLine: | + #!/bin/sh + echo "$(date -Iseconds): ----------------------------------------" + echo "$(date -Iseconds): Running ${GL_TOOLS_DIR}/init_tools.sh with output written to ${GL_WORKSPACE_LOGS_DIR}/init-tools.log..." + "${GL_TOOLS_DIR}/init_tools.sh" >> "${GL_WORKSPACE_LOGS_DIR}/init-tools.log" 2>&1 & + echo "$(date -Iseconds): Finished running ${GL_TOOLS_DIR}/init_tools.sh." + component: tooling-container + - id: gl-sleep-until-container-is-running-command + exec: + commandLine: | + #!/bin/sh + echo "$(date -Iseconds): ----------------------------------------" + echo "$(date -Iseconds): Sleeping until workspace is running..." + time_to_sleep=5 + status_file="/.workspace-data/variables/file/gl_workspace_reconciled_actual_state.txt" + while [ "$(cat ${status_file})" != "Running" ]; do + echo "$(date -Iseconds): Workspace state is '$(cat ${status_file})' from status file '${status_file}'. Blocking remaining postStart events execution for ${time_to_sleep} seconds until state is 'Running'..." + sleep ${time_to_sleep} + done + echo "$(date -Iseconds): Workspace state is now 'Running', continuing postStart hook execution." + echo "$(date -Iseconds): Finished sleeping until workspace is running." + component: tooling-container + - id: gl-project-cloner-command + apply: + component: gl-project-cloner + events: + preStart: + - gl-tools-injector-command + - gl-project-cloner-command + postStart: + - gl-start-sshd-command + - gl-init-tools-command + - gl-sleep-until-container-is-running-command + variables: {} diff --git a/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..675cbfff1c74bd99534eac08054915e5033eff89 --- /dev/null +++ b/ee/spec/lib/ee/gitlab/background_migration/backfill_workspace_agentk_states_spec.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::BackgroundMigration::BackfillWorkspaceAgentkStates, feature_category: :workspaces do + let(:organization) { table(:organizations).create!(name: "test-org", path: "default") } + let(:processed_devfile) do + read_fixture_file("processed_devfile.yaml") + end + + let!(:workspace_1) do + table(:workspaces).create!( + user_id: user.id, + project_id: project.id, + cluster_agent_id: agent.id, + desired_state_updated_at: Time.now.utc, + actual_state_updated_at: Time.now.utc, + responded_to_agent_at: Time.now.utc, + name: "workspace-1", + namespace: "workspace_1_namespace", + desired_state: "Terminated", + actual_state: "Terminated", + project_ref: "devfile-ref", + devfile_path: "devfile-path", + devfile: devfile, + processed_devfile: processed_devfile, + url: "workspace-url", + deployment_resource_version: "v1", + personal_access_token_id: personal_access_token.id, + workspaces_agent_config_version: agent_config_version.id, + desired_config_generator_version: 3 + ) + end + + let!(:workspace_2) do + table(:workspaces).create!( + user_id: user.id, + project_id: project.id, + cluster_agent_id: agent.id, + desired_state_updated_at: Time.now.utc, + actual_state_updated_at: Time.now.utc, + responded_to_agent_at: Time.now.utc, + name: "workspace-2", + namespace: "workspace_2_namespace", + desired_state: "Running", + actual_state: "Running", + project_ref: "devfile-ref", + devfile_path: "devfile-path", + devfile: devfile, + processed_devfile: processed_devfile, + url: "workspace-url", + deployment_resource_version: "v1", + personal_access_token_id: personal_access_token.id, + workspaces_agent_config_version: agent_config_version.id, + desired_config_generator_version: 3 + ) + end + + let(:migration) do + described_class.new( + start_id: workspace_1.id, + end_id: workspace_2.id, + batch_table: :workspaces, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + let(:expected_desired_config_workspace_1) do + json_str = read_fixture_file("desired_config.json") + json_str.gsub!("$WORKSPACE_ID", workspace_1.id.to_s) + json_str.gsub!("$WORKSPACE_NAMESPACE", workspace_1.namespace) + json_str.gsub!("$WORKSPACE_NAME", workspace_1.name) + json_str.gsub!("$REPLICA", "0") + ::Gitlab::Json.parse(json_str) + end + + let(:expected_desired_config_workspace_2) do + json_str = read_fixture_file("desired_config.json") + json_str.gsub!("$WORKSPACE_ID", workspace_2.id.to_s) + json_str.gsub!("$WORKSPACE_NAMESPACE", workspace_2.namespace) + json_str.gsub!("$WORKSPACE_NAME", workspace_2.name) + json_str.gsub!("$REPLICA", "1") + ::Gitlab::Json.parse(json_str) + end + + let(:user) do + table(:users).create!( + name: "test", + email: "test@example.com", + projects_limit: 5, + organization_id: organization.id + ) + end + + let(:namespace) { table(:namespaces).create!(name: "name", path: "path", organization_id: organization.id) } + let(:project) do + table(:projects).create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + organization_id: organization.id + ) + end + + let!(:personal_access_token) do + table(:personal_access_tokens).create!( + user_id: user.id, + name: "token_name", + expires_at: Time.now.utc, + organization_id: organization.id + ) + end + + let(:agent) do + table(:cluster_agents).create!( + id: 1, + name: "Agent-1", + project_id: project.id + ) + end + + let!(:agent_config) do + table(:workspaces_agent_configs).create!( + cluster_agent_id: agent.id, + enabled: true, + dns_zone: "test.workspace.me", + project_id: project.id + ) + end + + let!(:agent_config_version) do + table(:workspaces_agent_config_versions).create!( + project_id: project.id, + item_id: agent_config.id, + item_type: "Gitlab::BackgroundMigration::RemoteDevelopment::BMWorkspacesAgentConfig", + event: "create" + ) + end + + let(:devfile) do + <<~YAML + schemaVersion: 2.2.0 + components: + - name: tooling-container + attributes: + gl/inject-editor: true + container: + image: registry.gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/debian-bullseye-ruby-3.2-node-18.12:rubygems-3.4-git-2.33-lfs-2.9-yarn-1.22-graphicsmagick-1.3.36-gitlab-workspaces + env: + - name: KEY + value: VALUE + endpoints: + - name: http-3000 + targetPort: 3000 + YAML + end + + # @param [String] filename + # @return [String] + def read_fixture_file(filename) + File.read(Rails.root.join("ee/spec/fixtures/remote_development/background_migration", filename).to_s) + end + + context "when desired_config is valid" do + it "creates a record workspace_agentk_states table for each workspace", :unlimited_max_formatted_output_length do + expect { migration.perform } + .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } + .by(2) + + workspace_1_agentk_state_post_migration = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) + workspace_2_agentk_state_post_migration = table(:workspace_agentk_states).find_by(workspace_id: workspace_2.id) + + expect(workspace_1_agentk_state_post_migration.project_id).to eq(workspace_1.project_id) + expect(workspace_2_agentk_state_post_migration.project_id).to eq(workspace_2.project_id) + + expect(workspace_1_agentk_state_post_migration.desired_config).to eq(expected_desired_config_workspace_1) + expect(workspace_2_agentk_state_post_migration.desired_config).to eq(expected_desired_config_workspace_2) + end + end + + context "when desired_config is invalid" do + it "creates a record in workspace_agentk_states with failed message and terminates the workspace", + :unlimited_max_formatted_output_length do + error = Devfile::CliError.new( + "quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'" + ) + allow(error).to receive(:backtrace).and_return([ + "/app/lib/some_file.rb:123:in `method_name'", + "/app/lib/another_file.rb:456:in `another_method'" + ]) + + allow(Gitlab::BackgroundMigration::RemoteDevelopment::WorkspaceOperations::Create::DesiredConfig::BmMain) + .to receive(:main).and_raise(error) + + expect { migration.perform } + .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } + .by(2) + + workspace_1_updated = table(:workspaces).find(workspace_1.id) + workspace_2_updated = table(:workspaces).find(workspace_2.id) + + expect(workspace_1_updated.actual_state).to eq("Terminated") + expect(workspace_1_updated.desired_state).to eq("Terminated") + expect(workspace_2_updated.actual_state).to eq("Terminated") + expect(workspace_2_updated.desired_state).to eq("Terminated") + + saved_records = Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.all + saved_records.each do |record| + expect(record.desired_config).to include( + { + "error_message" => "quantities must match the regular expression " \ + "'^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'", + "message" => "Migration failed for this workspace. This workspace will be orphaned, cluster " \ + "administrators are advised to clean up the orphan workspaces.", + "backtrace" => [ + "/app/lib/some_file.rb:123:in `method_name'", + "/app/lib/another_file.rb:456:in `another_method'" + ] + } + ) + end + end + end + + context "when config already exists for a workspace" do + it "skips updating the config for that workspace", :unlimited_max_formatted_output_length do + workspace_2_agentk_state = table(:workspace_agentk_states).create!( + desired_config: { "some_key" => "some_value" }, + workspace_id: workspace_2.id, + project_id: workspace_2.project_id + ) + + expect(workspace_2_agentk_state).to be_persisted + + workspace_1_agentk_state = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) + expect(workspace_1_agentk_state).to be_nil + + expect { migration.perform } + .to change { Gitlab::BackgroundMigration::RemoteDevelopment::Models::BmWorkspaceAgentkState.count } + .by(1) + + workspace_1_agentk_state = table(:workspace_agentk_states).find_by(workspace_id: workspace_1.id) + expect(workspace_1_agentk_state).not_to be_nil + end + end +end diff --git a/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb b/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..9cbad4fd56a77210710e3960b4e77dc800291f3a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_workspace_agentk_states.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillWorkspaceAgentkStates < BatchedMigrationJob + operation_name :backfill_workspace_agentk_states + feature_category :workspaces + + # @return [Void] + def perform; end + end + end +end + +Gitlab::BackgroundMigration::BackfillWorkspaceAgentkStates.prepend_mod diff --git a/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb b/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb index e9001f87e952c1d4629ea343ccd0867a96a70126..267f0cc37c5df5fe23b0e91a01c169243592213d 100644 --- a/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb +++ b/lib/gitlab/background_migration/remote_development/bm_create_desired_config.rb @@ -28,12 +28,13 @@ def self.create_and_save(workspace_id:, dry_run: false) ) end + # rubocop:disable Metrics/MethodLength -- need it big # @param [BackgroundMigration::RemoteDevelopment::Models::BMWorkspace] workspace # @param [BackgroundMigration::RemoteDevelopment::WorkspaceOperations::BMDesiredConfig] desired_config # @param [Gitlab::BackgroundMigration::Logger] logger # @param [Boolean] dry_run # @return [Void] - def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, logger:, dry_run:) # rubocop:disable Metrics/MethodLength -- need it big + def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, logger:, dry_run:) if dry_run puts "For workspace_id #{workspace.id}" puts "Valid desired_config? #{desired_config.valid?}" @@ -54,6 +55,7 @@ def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, end if dry_run + # noinspection RubyArgCount -- RubyMine does not recognize the fields workspace_agentk_state = RemoteDevelopment::Models::BmWorkspaceAgentkState.new( workspace_id: workspace.id, project_id: workspace.project_id, @@ -65,17 +67,21 @@ def self.validate_and_create_workspace_agentk_state(workspace:, desired_config:, puts message end else - RemoteDevelopment::Models::BmWorkspaceAgentkState.create!( - workspace_id: workspace.id, - project_id: workspace.project_id, - desired_config: desired_config + RemoteDevelopment::Models::BmWorkspaceAgentkState.upsert( + { + workspace_id: workspace.id, + project_id: workspace.project_id, + desired_config: desired_config + }, + unique_by: :workspace_id ) end end + # rubocop:enable Metrics/MethodLength # @return [Gitlab::BackgroundMigration::Logger] def self.logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + @logger ||= ::Gitlab::BackgroundMigration::Logger.build end end end diff --git a/lib/gitlab/background_migration/remote_development/models/bm_agent.rb b/lib/gitlab/background_migration/remote_development/models/bm_agent.rb new file mode 100644 index 0000000000000000000000000000000000000000..279bf3d5ab8b4ca408dc6a90c1e5dd4ad39fdccb --- /dev/null +++ b/lib/gitlab/background_migration/remote_development/models/bm_agent.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module RemoteDevelopment + module Models + # rubocop:disable Migration/BatchedMigrationBaseClass -- This is not a migration file class so we do not need to inherit from BatchedMigrationJob + class BmAgent < ::Gitlab::Database::Migration[2.3]::MigrationRecord + self.table_name = 'cluster_agents' + + has_one :unversioned_latest_workspaces_agent_config, + class_name: 'RemoteDevelopment::Models::BmWorkspaceAgentConfig', + inverse_of: :agent, + foreign_key: :cluster_agent_id + + has_many :workspaces, + class_name: 'RemoteDevelopment::Models::BmWorkspace', + inverse_of: :agent, + foreign_key: :cluster_agent_id + end + + # rubocop:enable Migration/BatchedMigrationBaseClass + end + end + end +end diff --git a/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb b/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb index 35f8c33ddfb1a7cb0004a27f49607323a7a159c8..a3a4d6871b31a412d97ce528f0ec2118aa9bd20f 100644 --- a/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb +++ b/lib/gitlab/background_migration/remote_development/models/bm_workspace.rb @@ -10,7 +10,7 @@ class BmWorkspace < ::Gitlab::Database::Migration[2.3]::MigrationRecord self.table_name = 'workspaces' - belongs_to :agent, class_name: "Clusters::Agent", foreign_key: "cluster_agent_id", inverse_of: :workspaces + belongs_to :agent, class_name: "BmAgent", foreign_key: "cluster_agent_id", inverse_of: :workspaces # @return [Boolean] def desired_state_running? diff --git a/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb b/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb index 2cdf5252bd500707ef7687dbd1f64cf465ea94ce..3dec2d9e9d960b0977d36c5aa8ee86b9ad18655e 100644 --- a/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb +++ b/lib/gitlab/background_migration/remote_development/models/bm_workspace_agent_config.rb @@ -8,10 +8,10 @@ module Models class BmWorkspaceAgentConfig < ::Gitlab::Database::Migration[2.3]::MigrationRecord include WorkspaceOperations::BmStates - self.table_name = 'workspace_agent_configs' + self.table_name = 'workspaces_agent_configs' belongs_to :agent, - class_name: 'Clusters::Agent', foreign_key: 'cluster_agent_id', + class_name: 'BmAgent', foreign_key: 'cluster_agent_id', inverse_of: :unversioned_latest_workspaces_agent_config end # rubocop:enable Migration/BatchedMigrationBaseClass diff --git a/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb b/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..49fc46d1815c8fa64129acd8553e2d5cc4fdb070 --- /dev/null +++ b/spec/migrations/20250715102036_queue_backfill_workspace_agentk_states_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillWorkspaceAgentkStates, migration: :gitlab_main_org, feature_category: :workspaces do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main_org, + table_name: :workspaces, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end