From 9b8072a6f0074e0cebb156311976c6c0c0841e6e Mon Sep 17 00:00:00 2001 From: Chad Woolley Date: Mon, 3 Feb 2025 22:47:57 -0800 Subject: [PATCH] Workspaces: move file contents from heredoc to files - Move file contents from inline constants to separate files read at runtime - See https://gitlab.com/gitlab-org/gitlab/-/issues/511503 --- ee/lib/remote_development/README.md | 17 +++++ ee/lib/remote_development/files.rb | 37 +++++++++++ .../settings/default_devfile.yaml | 7 ++ .../settings/default_settings.rb | 23 ++----- .../create/create_constants.rb | 4 +- .../create/main_component_updater.rb | 25 +++---- .../main_component_updater_container_args.sh | 8 +++ .../project_cloner_component_inserter.rb | 43 ++++-------- ...loner_component_inserter_container_args.sh | 23 +++++++ .../create/workspace_creator.rb | 5 +- .../create/workspace_variables.rb | 32 ++------- ...e_variables_git_credential_store_script.sh | 18 +++++ .../remote_development/workspaces.rb | 3 +- .../example.default_devfile.yaml | 7 ++ ...xample.main-container-updated-devfile.yaml | 6 +- ...-updated-marketplace-disabled-devfile.yaml | 6 +- .../example.processed-devfile-v2.yaml | 15 ++--- .../example.processed-devfile.yaml | 15 ++--- ...ample.project-cloner-inserted-devfile.yaml | 15 ++--- .../settings/settings_initializer_spec.rb | 6 +- .../create/devfile_fetcher_spec.rb | 2 +- .../create/main_integration_spec.rb | 2 +- .../create/workspace_variables_spec.rb | 6 +- .../remote_development/integration_spec.rb | 4 +- .../remote_development_shared_contexts.rb | 65 ++++++++++--------- 25 files changed, 220 insertions(+), 174 deletions(-) create mode 100644 ee/lib/remote_development/files.rb create mode 100644 ee/lib/remote_development/settings/default_devfile.yaml create mode 100644 ee/lib/remote_development/workspace_operations/create/main_component_updater_container_args.sh create mode 100644 ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter_container_args.sh create mode 100644 ee/lib/remote_development/workspace_operations/create/workspace_variables_git_credential_store_script.sh create mode 100644 ee/spec/fixtures/remote_development/example.default_devfile.yaml diff --git a/ee/lib/remote_development/README.md b/ee/lib/remote_development/README.md index d05e72e22469b3..140900397ee95b 100644 --- a/ee/lib/remote_development/README.md +++ b/ee/lib/remote_development/README.md @@ -332,6 +332,23 @@ We _currently_ do not make heavy use of metaprogramming. But, we may make use of - Spec/factory code may refer to constants from different modules than their own if necessary. The scoping rules only apply to usages within production code. - Do not declare string constants which are only interpolated into other string constants, and are not used anywhere else. Instead, declare a single constant as single string. +### Using non-Ruby files in domain code + +The `RemoteDevelopment::Files` module contains constants for all the files (default devfile, shell scripts, script fragments, commands, etc) +that are used in the Remote Development domain for various purposes, such as programatically injecting into container `ENTRYPOINT`/`CMD`, etc. +For example: `RemoteDevelopment::Files::MAIN_COMPONENT_UPDATER_CONTAINER_ARGS`. + +The contents of these files are pulled out to separate files in the filesystem, instead of being hardcoded +via inline Ruby HEREDOC or other means. This allows them to have full support for syntax highlighting and refactoring in IDEs, +glob-based linting in CI (for example, running Shellcheck on all `**/*.sh`), etc. + +The files themselves should ideally be placed close to the code which uses them, and use a similar filename, so that the file list +shows them right next to the code that uses them. + +For example, the file with the shell script injected into the main container is located at +`ee/lib/remote_development/workspace_operations/create/workspace_variables_git_credential_store_script.sh`, and named +with the `workspace_variables_` prefix, because it is used in`ee/lib/remote_development/workspace_operations/create/devfile_fetcher.rb`. + ## Railway Oriented Programming and the Result class The Domain Logic layer uses the "Railway Oriented Programming" pattern (AKA "ROP"), which is [explained here in a presentation and video](https://fsharpforfunandprofit.com/rop/) by Scott Wlaschin. The presentation slides on that page give an overview which explains the motivation and implementation of this pattern. diff --git a/ee/lib/remote_development/files.rb b/ee/lib/remote_development/files.rb new file mode 100644 index 00000000000000..3add8f0be262a9 --- /dev/null +++ b/ee/lib/remote_development/files.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module RemoteDevelopment + # This module contains constants for all the files (default devfile, shell scripts, script fragments, commands, etc) + # that are used in the Remote Development domain. They are pulled out to separate files instead of being hardcoded + # via inline HEREDOC or other means, so that they can have full support for + # syntax highlighting, refactoring, linting, etc. + module Files + # @param [String] path - file path relative to domain logic root (this directory, `ee/lib/remote_development`) + # @return [String] content of the file + def self.read_file(path) + File.read("#{__dir__}/#{path}") + end + + #################################### + # Please keep this list alphabetized + #################################### + + # When updating DEFAULT_DEVFILE_YAML contents in `default_devfile.yaml`, update the user facing doc as well. + # https://docs.gitlab.com/ee/user/workspace/#devfault-devfile + # + # The container image is pinned to linux/amd64 digest, instead of the tag digest. + # This is to prevent Rancher Desktop from pulling the linux/arm64 architecture of the image + # which will disrupt local development since gitlab-workspaces-tools does not support + # that architecture yet and thus the workspace won't start. + # This will be fixed in https://gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools/-/issues/12 + DEFAULT_DEVFILE_YAML = read_file("settings/default_devfile.yaml") + PROJECTS_CLONER_COMPONENT_INSERTER_CONTAINER_ARGS = + read_file("workspace_operations/create/project_cloner_component_inserter_container_args.sh") + MAIN_COMPONENT_UPDATER_CONTAINER_ARGS = + read_file("workspace_operations/create/main_component_updater_container_args.sh") + WORKSPACE_VARIABLES_GIT_CREDENTIAL_STORE_SCRIPT = + read_file("workspace_operations/create/workspace_variables_git_credential_store_script.sh") + + private_class_method :read_file + end +end diff --git a/ee/lib/remote_development/settings/default_devfile.yaml b/ee/lib/remote_development/settings/default_devfile.yaml new file mode 100644 index 00000000000000..980af68390fe26 --- /dev/null +++ b/ee/lib/remote_development/settings/default_devfile.yaml @@ -0,0 +1,7 @@ +schemaVersion: "%s" +components: + - name: development-environment + attributes: + gl/inject-editor: true + container: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images/workspaces/ubuntu-24.04:20250109224147-golang-1.23@sha256:c3d5527641bc0c6f4fbbea4bb36fe225b8e9f1df69f682c927941327312bc676" diff --git a/ee/lib/remote_development/settings/default_settings.rb b/ee/lib/remote_development/settings/default_settings.rb index eac35fd68af5cd..b1c806048701b4 100644 --- a/ee/lib/remote_development/settings/default_settings.rb +++ b/ee/lib/remote_development/settings/default_settings.rb @@ -3,28 +3,11 @@ module RemoteDevelopment module Settings class DefaultSettings + include Files include RemoteDevelopmentConstants UNDEFINED = nil - # When updating DEFAULT_DEVFILE_YAML, update the user facing doc as well. - # https://docs.gitlab.com/ee/user/workspace/#devfault-devfile - # - # The container image is pinned to linux/amd64 digest, instead of the tag digest. - # This is to prevent Rancher Desktop from pulling the linux/arm64 architecture of the image - # which will disrupt local development since gitlab-workspaces-tools does not support - # that architecture yet and thus the workspace won't start. - # This will be fixed in https://gitlab.com/gitlab-org/workspaces/gitlab-workspaces-tools/-/issues/12 - DEFAULT_DEVFILE_YAML = <<~DEVFILE.freeze - schemaVersion: #{REQUIRED_DEVFILE_SCHEMA_VERSION} - components: - - name: development-environment - attributes: - gl/inject-editor: true - container: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images/workspaces/ubuntu-24.04:20250109224147-golang-1.23@sha256:c3d5527641bc0c6f4fbbea4bb36fe225b8e9f1df69f682c927941327312bc676" - DEVFILE - # ALL REMOTE DEVELOPMENT SETTINGS MUST BE DECLARED HERE. # See ../README.md for more details. # @return [Hash] @@ -36,7 +19,9 @@ def self.default_settings # the logic for reading settings from ::Gitlab::CurrentSettings. It can be replaced when there is an # actual Remote Development entry in ::Gitlab::CurrentSettings. default_branch_name: [UNDEFINED, String], - default_devfile_yaml: [DEFAULT_DEVFILE_YAML, String], + default_devfile_yaml: [ + format(DEFAULT_DEVFILE_YAML, schema_version: REQUIRED_DEVFILE_SCHEMA_VERSION), String + ], default_resources_per_workspace_container: [{}, Hash], default_runtime_class: ["", String], full_reconciliation_interval_seconds: [3600, Integer], diff --git a/ee/lib/remote_development/workspace_operations/create/create_constants.rb b/ee/lib/remote_development/workspace_operations/create/create_constants.rb index 0660e8f5720252..36bfaf6b83e25f 100644 --- a/ee/lib/remote_development/workspace_operations/create/create_constants.rb +++ b/ee/lib/remote_development/workspace_operations/create/create_constants.rb @@ -13,16 +13,16 @@ module CreateConstants include WorkspaceOperationsConstants # Please keep alphabetized - GIT_CREDENTIAL_STORE_FILE = "#{VARIABLES_FILE_DIR}/gl_git_credential_store.sh".freeze + GIT_CREDENTIAL_STORE_SCRIPT_FILE = "#{VARIABLES_FILE_DIR}/gl_git_credential_store.sh".freeze MAIN_COMPONENT_INDICATOR_ATTRIBUTE = "gl/inject-editor" NAMESPACE_PREFIX = "gl-rd-ns" - PROJECT_CLONER_COMPONENT_NAME = "gl-project-cloner" TOKEN_FILE = "#{VARIABLES_FILE_DIR}/gl_token".freeze TOOLS_DIR_NAME = ".gl-tools" TOOLS_DIR_ENV_VAR = "GL_TOOLS_DIR" TOOLS_INJECTOR_COMPONENT_NAME = "gl-tools-injector" WORKSPACE_DATA_VOLUME_NAME = "gl-workspace-data" WORKSPACE_DATA_VOLUME_PATH = "/projects" + WORKSPACE_EDITOR_PORT = 60001 end end end diff --git a/ee/lib/remote_development/workspace_operations/create/main_component_updater.rb b/ee/lib/remote_development/workspace_operations/create/main_component_updater.rb index 484765736d9d28..05d17348348df9 100644 --- a/ee/lib/remote_development/workspace_operations/create/main_component_updater.rb +++ b/ee/lib/remote_development/workspace_operations/create/main_component_updater.rb @@ -5,6 +5,9 @@ module WorkspaceOperations module Create class MainComponentUpdater include CreateConstants + include Files + + WORKSPACE_SSH_PORT = 60022 # @param [Hash] context # @return [Hash] @@ -15,9 +18,6 @@ def self.update(context) vscode_extensions_gallery_metadata: Hash => vscode_extensions_gallery_metadata } - editor_port = WorkspaceCreator::WORKSPACE_PORT - ssh_port = 60022 - # NOTE: We will always have exactly one main_component found, because we have already # validated this in post_flatten_devfile_validator.rb main_component = @@ -30,15 +30,15 @@ def self.update(context) update_env_vars( container: container, tools_dir: tools_dir, - editor_port: editor_port, - ssh_port: ssh_port, + editor_port: WORKSPACE_EDITOR_PORT, + ssh_port: WORKSPACE_SSH_PORT, enable_marketplace: vscode_extensions_gallery_metadata.fetch(:enabled) ) update_endpoints( container: container, - editor_port: editor_port, - ssh_port: ssh_port + editor_port: WORKSPACE_EDITOR_PORT, + ssh_port: WORKSPACE_SSH_PORT ) override_command_and_args( @@ -113,16 +113,7 @@ def self.override_command_and_args(container:) # This overrides the main container's command # Open issue to support both starting the editor and running the default command: # https://gitlab.com/gitlab-org/gitlab/-/issues/392853 - container_args = <<~"SH".chomp - sshd_path=$(which sshd) - if [ -x "$sshd_path" ]; then - echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & - else - echo "'sshd' not found in path. Not starting SSH server." - fi - ${GL_TOOLS_DIR}/init_tools.sh - SH + container_args = MAIN_COMPONENT_UPDATER_CONTAINER_ARGS container[:command] = %w[/bin/sh -c] container[:args] = [container_args] diff --git a/ee/lib/remote_development/workspace_operations/create/main_component_updater_container_args.sh b/ee/lib/remote_development/workspace_operations/create/main_component_updater_container_args.sh new file mode 100644 index 00000000000000..0b00f476a16a97 --- /dev/null +++ b/ee/lib/remote_development/workspace_operations/create/main_component_updater_container_args.sh @@ -0,0 +1,8 @@ +sshd_path=$(which sshd) +if [ -x "$sshd_path" ]; then + echo "Starting sshd on port ${GL_SSH_PORT}" + $sshd_path -D -p "${GL_SSH_PORT}" & +else + echo "'sshd' not found in path. Not starting SSH server." +fi +"${GL_TOOLS_DIR}/init_tools.sh" diff --git a/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter.rb b/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter.rb index ae117a2de3c9f2..4e3031549e0277 100644 --- a/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter.rb +++ b/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter.rb @@ -5,6 +5,10 @@ module WorkspaceOperations module Create class ProjectClonerComponentInserter include CreateConstants + include Files + + PROJECT_CLONER_COMPONENT_NAME = "gl-project-cloner" + PROJECT_CLONING_SUCCESSFUL_FILENAME = ".gl_project_cloning_successful" # @param [Hash] context # @return [Hash] @@ -24,7 +28,7 @@ def self.insert(context) settings => { project_cloner_image: String => image, } - project_cloning_successful_file = "#{volume_path}/.gl_project_cloning_successful" + project_cloning_successful_file = "#{volume_path}/#{PROJECT_CLONING_SUCCESSFUL_FILENAME}" # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/408448 # replace the alpine/git docker image with one that is published by gitlab for security / reliability @@ -40,31 +44,12 @@ def self.insert(context) # remove the directory before cloning. # Once cloning is successful, create the file which is used in the check above. # This will ensure the project is not cloned again on restarts. - container_args = <<~SH.chomp - if [ -f "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}" ]; - then - echo "Project cloning was already successful"; - exit 0; - fi - if [ -d "#{Shellwords.shellescape(clone_dir)}" ]; - then - echo "Removing unsuccessfully cloned project directory"; - rm -rf "#{Shellwords.shellescape(clone_dir)}"; - fi - echo "Cloning project"; - git clone --branch "#{Shellwords.shellescape(project_ref)}" "#{Shellwords.shellescape(project_url)}" "#{Shellwords.shellescape(clone_dir)}"; - exit_code=$? - if [ "${exit_code}" -eq 0 ]; - then - echo "Project cloning successful"; - touch "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}"; - echo "Updated file to indicate successful project cloning"; - exit 0; - else - echo "Project cloning failed with exit code: ${exit_code}"; - exit "${exit_code}"; - fi - SH + container_args = format(PROJECTS_CLONER_COMPONENT_INSERTER_CONTAINER_ARGS, + project_cloning_successful_file: Shellwords.shellescape(project_cloning_successful_file), + clone_dir: Shellwords.shellescape(clone_dir), + project_ref: Shellwords.shellescape(project_ref), + project_url: Shellwords.shellescape(project_url) + ) # TODO: https://gitlab.com/groups/gitlab-org/-/epics/10461 # implement better error handling to allow cloner to be able to deal with different categories of errors @@ -77,12 +62,6 @@ def self.insert(context) # command has been overridden here as the default command in the alpine/git # container invokes git directly command: %w[/bin/sh -c], - env: [ - { - name: "GL_PROJECT_CLONING_SUCCESSFUL_FILE", - value: project_cloning_successful_file - } - ], memoryLimit: "512Mi", memoryRequest: "256Mi", cpuLimit: "500m", diff --git a/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter_container_args.sh b/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter_container_args.sh new file mode 100644 index 00000000000000..f7251ea89121c4 --- /dev/null +++ b/ee/lib/remote_development/workspace_operations/create/project_cloner_component_inserter_container_args.sh @@ -0,0 +1,23 @@ +if [ -f "%s" ]; +then + echo "Project cloning was already successful"; + exit 0; +fi +if [ -d "%s" ]; +then + echo "Removing unsuccessfully cloned project directory"; + rm -rf "%s"; +fi +echo "Cloning project"; +git clone --branch "%s" "%s" "%s"; +exit_code=$? +if [ "${exit_code}" -eq 0 ]; +then + echo "Project cloning successful"; + touch "%s"; + echo "Updated file to indicate successful project cloning"; + exit 0; +else + echo "Project cloning failed with exit code: ${exit_code}"; + exit "${exit_code}"; +fi diff --git a/ee/lib/remote_development/workspace_operations/create/workspace_creator.rb b/ee/lib/remote_development/workspace_operations/create/workspace_creator.rb index d07ec627e528dd..994780257bdd0a 100644 --- a/ee/lib/remote_development/workspace_operations/create/workspace_creator.rb +++ b/ee/lib/remote_development/workspace_operations/create/workspace_creator.rb @@ -4,11 +4,10 @@ module RemoteDevelopment module WorkspaceOperations module Create class WorkspaceCreator + include CreateConstants include States include Messages - WORKSPACE_PORT = 60001 - # @param [Hash] context # @return [Gitlab::Fp::Result] def self.create(context) @@ -77,7 +76,7 @@ def self.create(context) # @param [String] project_dir # @return [void] def self.set_workspace_url(workspace:, agent_dns_zone:, project_dir:) - host = "#{WORKSPACE_PORT}-#{workspace.name}.#{agent_dns_zone}" + host = "#{WORKSPACE_EDITOR_PORT}-#{workspace.name}.#{agent_dns_zone}" query = { folder: project_dir }.to_query # NOTE: Use URI builder to ensure that we are building a valid URI, then retrieve parts from it diff --git a/ee/lib/remote_development/workspace_operations/create/workspace_variables.rb b/ee/lib/remote_development/workspace_operations/create/workspace_variables.rb index 194ceae28a9b9c..866489f12df6a0 100644 --- a/ee/lib/remote_development/workspace_operations/create/workspace_variables.rb +++ b/ee/lib/remote_development/workspace_operations/create/workspace_variables.rb @@ -5,27 +5,7 @@ module WorkspaceOperations module Create class WorkspaceVariables include CreateConstants - - GIT_CREDENTIAL_STORE_SCRIPT = <<~SH.chomp - #!/bin/sh - # This is a readonly store so we can exit cleanly when git attempts a store or erase action - if [ "$1" != "get" ]; - then - exit 0 - fi - - if [ -z "${GL_TOKEN_FILE_PATH}" ]; - then - echo "We could not find the GL_TOKEN_FILE_PATH variable" - exit 1 - fi - password=$(cat ${GL_TOKEN_FILE_PATH}) - - # The username is derived from the "user.email" configuration item. Ensure it is set. - echo "username=does-not-matter" - echo "password=${password}" - exit 0 - SH + include Files # @param [String] name # @param [String] dns_zone @@ -54,8 +34,8 @@ def self.variables( workspace_id: workspace_id }, { - key: File.basename(GIT_CREDENTIAL_STORE_FILE), - value: GIT_CREDENTIAL_STORE_SCRIPT, + key: File.basename(GIT_CREDENTIAL_STORE_SCRIPT_FILE), + value: WORKSPACE_VARIABLES_GIT_CREDENTIAL_STORE_SCRIPT, variable_type: RemoteDevelopment::Enums::Workspace::WORKSPACE_VARIABLE_TYPES[:file], workspace_id: workspace_id }, @@ -73,7 +53,7 @@ def self.variables( }, { key: 'GIT_CONFIG_VALUE_0', - value: GIT_CREDENTIAL_STORE_FILE, + value: GIT_CREDENTIAL_STORE_SCRIPT_FILE, variable_type: RemoteDevelopment::Enums::Workspace::WORKSPACE_VARIABLE_TYPES[:environment], workspace_id: workspace_id }, @@ -102,8 +82,8 @@ def self.variables( workspace_id: workspace_id }, { - key: 'GL_GIT_CREDENTIAL_STORE_FILE_PATH', - value: GIT_CREDENTIAL_STORE_FILE, + key: 'GL_GIT_CREDENTIAL_STORE_SCRIPT_FILE', + value: GIT_CREDENTIAL_STORE_SCRIPT_FILE, variable_type: RemoteDevelopment::Enums::Workspace::WORKSPACE_VARIABLE_TYPES[:environment], workspace_id: workspace_id }, diff --git a/ee/lib/remote_development/workspace_operations/create/workspace_variables_git_credential_store_script.sh b/ee/lib/remote_development/workspace_operations/create/workspace_variables_git_credential_store_script.sh new file mode 100644 index 00000000000000..704a9be35d0942 --- /dev/null +++ b/ee/lib/remote_development/workspace_operations/create/workspace_variables_git_credential_store_script.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# This is a readonly store so we can exit cleanly when git attempts a store or erase action +if [ "$1" != "get" ]; +then + exit 0 +fi + +if [ -z "${GL_TOKEN_FILE_PATH}" ]; +then + echo "We could not find the GL_TOKEN_FILE_PATH variable" + exit 1 +fi +password=$(cat "${GL_TOKEN_FILE_PATH}") + +# The username is derived from the "user.email" configuration item. Ensure it is set. +echo "username=does-not-matter" +echo "password=${password}" +exit 0 diff --git a/ee/spec/factories/remote_development/workspaces.rb b/ee/spec/factories/remote_development/workspaces.rb index 24ebf479263c2a..a503c8d8087384 100644 --- a/ee/spec/factories/remote_development/workspaces.rb +++ b/ee/spec/factories/remote_development/workspaces.rb @@ -58,7 +58,8 @@ user = workspace.user workspace.project.add_developer(user) workspace.agent.project.add_developer(user) - workspace.url_prefix ||= "60001-#{workspace.name}" + workspace.url_prefix ||= + "#{RemoteDevelopment::WorkspaceOperations::Create::CreateConstants::WORKSPACE_EDITOR_PORT}-#{workspace.name}" workspace.url_query_string ||= "folder=dir%2Ffile" end diff --git a/ee/spec/fixtures/remote_development/example.default_devfile.yaml b/ee/spec/fixtures/remote_development/example.default_devfile.yaml new file mode 100644 index 00000000000000..316bc0b098ee82 --- /dev/null +++ b/ee/spec/fixtures/remote_development/example.default_devfile.yaml @@ -0,0 +1,7 @@ +schemaVersion: "2.2.0" +components: + - name: development-environment + attributes: + gl/inject-editor: true + container: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images/workspaces/ubuntu-24.04:20250109224147-golang-1.23@sha256:c3d5527641bc0c6f4fbbea4bb36fe225b8e9f1df69f682c927941327312bc676" diff --git a/ee/spec/fixtures/remote_development/example.main-container-updated-devfile.yaml b/ee/spec/fixtures/remote_development/example.main-container-updated-devfile.yaml index 97fbc7cef75c63..f332cdd68c80fb 100644 --- a/ee/spec/fixtures/remote_development/example.main-container-updated-devfile.yaml +++ b/ee/spec/fixtures/remote_development/example.main-container-updated-devfile.yaml @@ -8,15 +8,15 @@ components: container: image: quay.io/mloriedo/universal-developer-image:ubi8-dw-demo args: - - |- + - | sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" command: - "/bin/sh" - "-c" diff --git a/ee/spec/fixtures/remote_development/example.main-container-updated-marketplace-disabled-devfile.yaml b/ee/spec/fixtures/remote_development/example.main-container-updated-marketplace-disabled-devfile.yaml index 97fbc7cef75c63..f332cdd68c80fb 100644 --- a/ee/spec/fixtures/remote_development/example.main-container-updated-marketplace-disabled-devfile.yaml +++ b/ee/spec/fixtures/remote_development/example.main-container-updated-marketplace-disabled-devfile.yaml @@ -8,15 +8,15 @@ components: container: image: quay.io/mloriedo/universal-developer-image:ubi8-dw-demo args: - - |- + - | sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" command: - "/bin/sh" - "-c" diff --git a/ee/spec/fixtures/remote_development/example.processed-devfile-v2.yaml b/ee/spec/fixtures/remote_development/example.processed-devfile-v2.yaml index d878ad18e75b85..7881b352c30563 100644 --- a/ee/spec/fixtures/remote_development/example.processed-devfile-v2.yaml +++ b/ee/spec/fixtures/remote_development/example.processed-devfile-v2.yaml @@ -8,15 +8,15 @@ components: container: image: quay.io/mloriedo/universal-developer-image:ubi8-dw-demo args: - - |- + - | sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" command: - "/bin/sh" - "-c" @@ -76,12 +76,9 @@ components: volumeMounts: - name: gl-workspace-data path: "/projects" - env: - - name: GL_PROJECT_CLONING_SUCCESSFUL_FILE - value: "/projects/.gl_project_cloning_successful" args: - - |- - if [ -f "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}" ]; + - | + if [ -f "/projects/.gl_project_cloning_successful" ]; then echo "Project cloning was already successful"; exit 0; @@ -97,7 +94,7 @@ components: if [ "${exit_code}" -eq 0 ]; then echo "Project cloning successful"; - touch "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}"; + touch "/projects/.gl_project_cloning_successful"; echo "Updated file to indicate successful project cloning"; exit 0; else diff --git a/ee/spec/fixtures/remote_development/example.processed-devfile.yaml b/ee/spec/fixtures/remote_development/example.processed-devfile.yaml index d878ad18e75b85..7881b352c30563 100644 --- a/ee/spec/fixtures/remote_development/example.processed-devfile.yaml +++ b/ee/spec/fixtures/remote_development/example.processed-devfile.yaml @@ -8,15 +8,15 @@ components: container: image: quay.io/mloriedo/universal-developer-image:ubi8-dw-demo args: - - |- + - | sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" command: - "/bin/sh" - "-c" @@ -76,12 +76,9 @@ components: volumeMounts: - name: gl-workspace-data path: "/projects" - env: - - name: GL_PROJECT_CLONING_SUCCESSFUL_FILE - value: "/projects/.gl_project_cloning_successful" args: - - |- - if [ -f "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}" ]; + - | + if [ -f "/projects/.gl_project_cloning_successful" ]; then echo "Project cloning was already successful"; exit 0; @@ -97,7 +94,7 @@ components: if [ "${exit_code}" -eq 0 ]; then echo "Project cloning successful"; - touch "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}"; + touch "/projects/.gl_project_cloning_successful"; echo "Updated file to indicate successful project cloning"; exit 0; else diff --git a/ee/spec/fixtures/remote_development/example.project-cloner-inserted-devfile.yaml b/ee/spec/fixtures/remote_development/example.project-cloner-inserted-devfile.yaml index 1c00a7d0e092ac..9b3620292796b3 100644 --- a/ee/spec/fixtures/remote_development/example.project-cloner-inserted-devfile.yaml +++ b/ee/spec/fixtures/remote_development/example.project-cloner-inserted-devfile.yaml @@ -8,15 +8,15 @@ components: container: image: quay.io/mloriedo/universal-developer-image:ubi8-dw-demo args: - - |- + - | sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" command: - "/bin/sh" - "-c" @@ -64,12 +64,9 @@ components: - name: gl-project-cloner container: image: alpine/git:2.45.2 - env: - - name: GL_PROJECT_CLONING_SUCCESSFUL_FILE - value: "/projects/.gl_project_cloning_successful" args: - - |- - if [ -f "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}" ]; + - | + if [ -f "/projects/.gl_project_cloning_successful" ]; then echo "Project cloning was already successful"; exit 0; @@ -85,7 +82,7 @@ components: if [ "${exit_code}" -eq 0 ]; then echo "Project cloning successful"; - touch "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}"; + touch "/projects/.gl_project_cloning_successful"; echo "Updated file to indicate successful project cloning"; exit 0; else diff --git a/ee/spec/lib/remote_development/settings/settings_initializer_spec.rb b/ee/spec/lib/remote_development/settings/settings_initializer_spec.rb index bdca65c874939b..c19e9df0698793 100644 --- a/ee/spec/lib/remote_development/settings/settings_initializer_spec.rb +++ b/ee/spec/lib/remote_development/settings/settings_initializer_spec.rb @@ -10,7 +10,9 @@ { requested_setting_names: requested_setting_names } end - let(:default_devfile_yaml) { ::RemoteDevelopment::Settings::DefaultSettings::DEFAULT_DEVFILE_YAML } + let(:default_devfile_yaml) do + File.read(Rails.root.join("ee/spec/fixtures/remote_development/example.default_devfile.yaml").to_s) + end subject(:returned_value) do described_class.init(context) @@ -86,7 +88,7 @@ max_resources_per_workspace: Hash, max_stopped_hours_before_termination: Integer, network_policy_egress: Array, - network_policy_enabled: Object, + network_policy_enabled: :Boolean, partial_reconciliation_interval_seconds: Integer, project_cloner_image: String, tools_injector_image: String, diff --git a/ee/spec/lib/remote_development/workspace_operations/create/devfile_fetcher_spec.rb b/ee/spec/lib/remote_development/workspace_operations/create/devfile_fetcher_spec.rb index 42fb19af1d2e81..adb6ebf74ca776 100644 --- a/ee/spec/lib/remote_development/workspace_operations/create/devfile_fetcher_spec.rb +++ b/ee/spec/lib/remote_development/workspace_operations/create/devfile_fetcher_spec.rb @@ -28,7 +28,7 @@ } end - let(:default_devfile_yaml) { ::RemoteDevelopment::Settings::DefaultSettings::DEFAULT_DEVFILE_YAML } + let(:default_devfile_yaml) { example_default_devfile_yaml } let(:context) do { diff --git a/ee/spec/lib/remote_development/workspace_operations/create/main_integration_spec.rb b/ee/spec/lib/remote_development/workspace_operations/create/main_integration_spec.rb index 0c1f802a074e0c..522ebc2aa79555 100644 --- a/ee/spec/lib/remote_development/workspace_operations/create/main_integration_spec.rb +++ b/ee/spec/lib/remote_development/workspace_operations/create/main_integration_spec.rb @@ -27,7 +27,7 @@ ] end - let(:default_devfile_yaml) { ::RemoteDevelopment::Settings::DefaultSettings::DEFAULT_DEVFILE_YAML } + let(:default_devfile_yaml) { example_default_devfile_yaml } let(:project) do files = devfile_path.nil? ? {} : { devfile_path => devfile_yaml } diff --git a/ee/spec/lib/remote_development/workspace_operations/create/workspace_variables_spec.rb b/ee/spec/lib/remote_development/workspace_operations/create/workspace_variables_spec.rb index 23386173597e5a..e9b666976c4ff9 100644 --- a/ee/spec/lib/remote_development/workspace_operations/create/workspace_variables_spec.rb +++ b/ee/spec/lib/remote_development/workspace_operations/create/workspace_variables_spec.rb @@ -13,7 +13,7 @@ let(:vscode_extensions_gallery_item_url) { "https://open-vsx.org/vscode/item" } let(:vscode_extensions_gallery_resource_url_template) { "https://open-vsx.org/vscode/asset/{publisher}/{name}/{version}/Microsoft.VisualStudio.Code.WebResources/{path}" } let(:git_credential_store_script) do - <<~SH.chomp + <<~SH #!/bin/sh # This is a readonly store so we can exit cleanly when git attempts a store or erase action if [ "$1" != "get" ]; @@ -26,7 +26,7 @@ echo "We could not find the GL_TOKEN_FILE_PATH variable" exit 1 fi - password=$(cat ${GL_TOKEN_FILE_PATH}) + password=$(cat "${GL_TOKEN_FILE_PATH}") # The username is derived from the "user.email" configuration item. Ensure it is set. echo "username=does-not-matter" @@ -92,7 +92,7 @@ workspace_id: workspace_id }, { - key: "GL_GIT_CREDENTIAL_STORE_FILE_PATH", + key: "GL_GIT_CREDENTIAL_STORE_SCRIPT_FILE", value: "/.workspace-data/variables/file/gl_git_credential_store.sh", variable_type: RemoteDevelopment::Enums::Workspace::WORKSPACE_VARIABLE_TYPES[:environment], workspace_id: workspace_id diff --git a/ee/spec/requests/remote_development/integration_spec.rb b/ee/spec/requests/remote_development/integration_spec.rb index ecea4e467364ba..cbd0b42a5f8978 100644 --- a/ee/spec/requests/remote_development/integration_spec.rb +++ b/ee/spec/requests/remote_development/integration_spec.rb @@ -98,12 +98,12 @@ { key: "GL_EDITOR_EXTENSIONS_GALLERY_ITEM_URL", type: :environment, value: "https://open-vsx.org/vscode/item" }, { key: "GL_EDITOR_EXTENSIONS_GALLERY_RESOURCE_URL_TEMPLATE", type: :environment, value: "https://open-vsx.org/vscode/asset/{publisher}/{name}/{version}/Microsoft.VisualStudio.Code.WebResources/{path}" }, { key: "GL_EDITOR_EXTENSIONS_GALLERY_SERVICE_URL", type: :environment, value: "https://open-vsx.org/vscode/gallery" }, - { key: "GL_GIT_CREDENTIAL_STORE_FILE_PATH", type: :environment, value: "/.workspace-data/variables/file/gl_git_credential_store.sh" }, + { key: "GL_GIT_CREDENTIAL_STORE_SCRIPT_FILE", type: :environment, value: "/.workspace-data/variables/file/gl_git_credential_store.sh" }, { key: "GL_TOKEN_FILE_PATH", type: :environment, value: "/.workspace-data/variables/file/gl_token" }, { key: "GL_WORKSPACE_DOMAIN_TEMPLATE", type: :environment, value: "${PORT}-workspace-#{agent.id}-#{user.id}-#{random_string}.#{dns_zone}" }, { key: "GITLAB_WORKFLOW_INSTANCE_URL", type: :environment, value: Gitlab::Routing.url_helpers.root_url }, { key: "GITLAB_WORKFLOW_TOKEN_FILE", type: :environment, value: "/.workspace-data/variables/file/gl_token" }, - { key: "gl_git_credential_store.sh", type: :file, value: RemoteDevelopment::WorkspaceOperations::Create::WorkspaceVariables::GIT_CREDENTIAL_STORE_SCRIPT }, + { key: "gl_git_credential_store.sh", type: :file, value: RemoteDevelopment::Files::WORKSPACE_VARIABLES_GIT_CREDENTIAL_STORE_SCRIPT }, { key: "gl_token", type: :file, value: /glpat-.+/ } ] # rubocop:enable Layout/LineLength diff --git a/ee/spec/support/shared_contexts/remote_development/remote_development_shared_contexts.rb b/ee/spec/support/shared_contexts/remote_development/remote_development_shared_contexts.rb index ec77ea9995a3a9..484132b5b3b7cb 100644 --- a/ee/spec/support/shared_contexts/remote_development/remote_development_shared_contexts.rb +++ b/ee/spec/support/shared_contexts/remote_development/remote_development_shared_contexts.rb @@ -56,7 +56,7 @@ def create_workspace_agent_info_hash( status = case [previous_actual_state, current_actual_state, workspace_exists] in [RemoteDevelopment::WorkspaceOperations::States::CREATION_REQUESTED, RemoteDevelopment::WorkspaceOperations::States::STARTING, _] - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:14:14Z" lastUpdateTime: "2023-04-10T10:14:14Z" @@ -64,9 +64,9 @@ def create_workspace_agent_info_hash( reason: NewReplicaSetCreated status: "True" type: Progressing - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::STARTING, false] - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:14:14Z" lastUpdateTime: "2023-04-10T10:14:14Z" @@ -84,9 +84,9 @@ def create_workspace_agent_info_hash( replicas: 1 unavailableReplicas: 1 updatedReplicas: 1 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::RUNNING, false] - <<~STATUS_YAML + <<~'YAML' availableReplicas: 1 conditions: - lastTransitionTime: "2023-03-06T14:36:36Z" @@ -104,7 +104,7 @@ def create_workspace_agent_info_hash( readyReplicas: 1 replicas: 1 updatedReplicas: 1 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::FAILED, false] raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError in [RemoteDevelopment::WorkspaceOperations::States::FAILED, RemoteDevelopment::WorkspaceOperations::States::STARTING, false] @@ -112,7 +112,7 @@ def create_workspace_agent_info_hash( in [RemoteDevelopment::WorkspaceOperations::States::RUNNING, RemoteDevelopment::WorkspaceOperations::States::FAILED, _] raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError in [RemoteDevelopment::WorkspaceOperations::States::RUNNING, RemoteDevelopment::WorkspaceOperations::States::STOPPING, _] - <<~STATUS_YAML + <<~'YAML' availableReplicas: 1 conditions: - lastTransitionTime: "2023-04-10T10:40:35Z" @@ -131,9 +131,9 @@ def create_workspace_agent_info_hash( readyReplicas: 1 replicas: 1 updatedReplicas: 1 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STOPPING, RemoteDevelopment::WorkspaceOperations::States::STOPPED, _] - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:40:35Z" lastUpdateTime: "2023-04-10T10:40:35Z" @@ -148,9 +148,9 @@ def create_workspace_agent_info_hash( status: "True" type: Progressing observedGeneration: 2 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STOPPED, RemoteDevelopment::WorkspaceOperations::States::STOPPED, true] - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:40:24Z" lastUpdateTime: "2023-04-10T10:40:35Z" @@ -165,13 +165,13 @@ def create_workspace_agent_info_hash( status: "True" type: Progressing observedGeneration: 2 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STOPPING, RemoteDevelopment::WorkspaceOperations::States::FAILED, _] raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError in [RemoteDevelopment::WorkspaceOperations::States::STOPPED, RemoteDevelopment::WorkspaceOperations::States::STARTING, _] # There are multiple state transitions inside kubernetes # Fields like `replicas`, `unavailableReplicas` and `updatedReplicas` eventually become present - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:40:24Z" lastUpdateTime: "2023-04-10T10:40:35Z" @@ -186,7 +186,7 @@ def create_workspace_agent_info_hash( status: "False" type: Available observedGeneration: 3 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STOPPED, RemoteDevelopment::WorkspaceOperations::States::FAILED, _] # Stopped workspace is terminated by the user which results in a Failed actual state. # e.g. could not unmount volume and terminate the workspace @@ -194,7 +194,7 @@ def create_workspace_agent_info_hash( in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::STARTING, true] # There are multiple state transitions inside kubernetes # Fields like `replicas`, `unavailableReplicas` and `updatedReplicas` eventually become present - <<~STATUS_YAML + <<~'YAML' conditions: - lastTransitionTime: "2023-04-10T10:40:24Z" lastUpdateTime: "2023-04-10T10:40:35Z" @@ -212,9 +212,9 @@ def create_workspace_agent_info_hash( replicas: 1 unavailableReplicas: 1 updatedReplicas: 1 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::RUNNING, true] - <<~STATUS_YAML + <<~'YAML' availableReplicas: 1 conditions: - lastTransitionTime: "2023-04-10T10:40:24Z" @@ -233,7 +233,7 @@ def create_workspace_agent_info_hash( readyReplicas: 1 replicas: 1 updatedReplicas: 1 - STATUS_YAML + YAML in [RemoteDevelopment::WorkspaceOperations::States::STARTING, RemoteDevelopment::WorkspaceOperations::States::FAILED, true] raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError in [RemoteDevelopment::WorkspaceOperations::States::FAILED, RemoteDevelopment::WorkspaceOperations::States::STARTING, true] @@ -242,7 +242,7 @@ def create_workspace_agent_info_hash( raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError in [_, RemoteDevelopment::WorkspaceOperations::States::FAILED, _] raise RemoteDevelopment::AgentInfoStatusFixtureNotImplementedError - # <<~STATUS_YAML + # <<~'YAML' # conditions: # - lastTransitionTime: "2023-03-06T14:36:31Z" # lastUpdateTime: "2023-03-08T11:16:35Z" @@ -259,7 +259,7 @@ def create_workspace_agent_info_hash( # replicas: 1 # unavailableReplicas: 1 # updatedReplicas: 1 - # STATUS_YAML + # YAML else msg = 'Unsupported state transition passed for create_workspace_agent_info_hash fixture creation: ' \ @@ -474,6 +474,7 @@ def create_config_to_apply_v3( yaml = YAML.dump(Gitlab::Utils.deep_sort_hash(resource).deep_stringify_keys) yaml.gsub!('test-project', project_name) yaml.gsub!('test-group', namespace_path) + yaml.gsub!('http://localhost/', root_url) yaml end.join end @@ -545,15 +546,15 @@ def workspace_deployment( containers: [ { args: [ - <<~"SH".chomp + <<~"SH" sshd_path=$(which sshd) if [ -x "$sshd_path" ]; then echo "Starting sshd on port ${GL_SSH_PORT}" - $sshd_path -D -p $GL_SSH_PORT & + $sshd_path -D -p "${GL_SSH_PORT}" & else echo "'sshd' not found in path. Not starting SSH server." fi - ${GL_TOOLS_DIR}/init_tools.sh + "${GL_TOOLS_DIR}/init_tools.sh" SH ], command: %w[/bin/sh -c], @@ -664,8 +665,8 @@ def workspace_deployment( initContainers: [ { args: [ - <<~ARGS.chomp - if [ -f "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}" ]; + <<~'SHELL' + if [ -f "/projects/.gl_project_cloning_successful" ]; then echo "Project cloning was already successful"; exit 0; @@ -676,26 +677,22 @@ def workspace_deployment( rm -rf "/projects/test-project"; fi echo "Cloning project"; - git clone --branch "master" "#{root_url}test-group/test-project.git" "/projects/test-project"; + git clone --branch "master" "http://localhost/test-group/test-project.git" "/projects/test-project"; exit_code=$? if [ "${exit_code}" -eq 0 ]; then echo "Project cloning successful"; - touch "${GL_PROJECT_CLONING_SUCCESSFUL_FILE}"; + touch "/projects/.gl_project_cloning_successful"; echo "Updated file to indicate successful project cloning"; exit 0; else echo "Project cloning failed with exit code: ${exit_code}"; exit "${exit_code}"; fi - ARGS + SHELL ], command: %w[/bin/sh -c], env: [ - { - name: "GL_PROJECT_CLONING_SUCCESSFUL_FILE", - value: "/projects/.gl_project_cloning_successful" - }, { name: "PROJECTS_ROOT", value: "/projects" @@ -1090,6 +1087,10 @@ def yaml_safe_load_symbolized(yaml) YAML.safe_load(yaml).to_h.deep_symbolize_keys end + def example_default_devfile_yaml + read_devfile_yaml('example.default_devfile.yaml') + end + def example_devfile_yaml read_devfile_yaml('example.devfile.yaml') end -- GitLab