From f71c9f6acd64c855af445eda89c045754810248d Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Thu, 24 Oct 2024 09:36:23 +0200 Subject: [PATCH 1/6] feat(release): "release create --publish-to-catalog" flag In this change, we are moving the logic of publishing CI components into `glab`. With this new flag; ``` glab release create 3.0.0 --publish-to-catalog ``` After creating the release, `glab` will fetch the components from the current project, parses the component names and their `spec`s, and sends a request to the `/api/v4/projects/:PID/catalog/publish` with a body. Delivers #7679 --- commands/release/create/create.go | 29 ++- commands/release/create/create_test.go | 135 +++++++++++++ .../release/create/publish_to_catalog/run.go | 181 ++++++++++++++++++ .../test-repo/templates/component-1.yml | 10 + .../test-repo/templates/component-2.yml | 3 + .../templates/component-3/template.yml | 10 + docs/source/release/create.md | 15 ++ docs/source/release/publish-to-catalog.md | 54 ++++++ 8 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 commands/release/create/publish_to_catalog/run.go create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml create mode 100644 docs/source/release/publish-to-catalog.md diff --git a/commands/release/create/create.go b/commands/release/create/create.go index a291dccb2..7767c4834 100644 --- a/commands/release/create/create.go +++ b/commands/release/create/create.go @@ -10,6 +10,7 @@ import ( "strings" "time" + publishtocatalog "gitlab.com/gitlab-org/cli/commands/release/create/publish_to_catalog" "gitlab.com/gitlab-org/cli/commands/release/releaseutils" "gitlab.com/gitlab-org/cli/commands/release/releaseutils/upload" @@ -41,6 +42,7 @@ type CreateOpts struct { AssetLinksAsJson string ReleasedAt string RepoOverride string + PublishToCatalog bool NoteProvided bool ReleaseNotesAction string @@ -78,7 +80,7 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { To fetch the new tag locally after the release, run %[1]sgit fetch --tags origin%[1]s. `, "`"), Args: cmdutils.MinimumArgs(1, "no tag name provided"), - Example: heredoc.Doc(` + Example: heredoc.Docf(` # Interactively create a release $ glab release create v1.0.1 @@ -110,7 +112,21 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { "direct_asset_path": "path/to/file" } ]' -`), + + # Create a release and publish it to the GitLab CI/CD catalog + # This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. + # The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. + # This command retrieves components from the current repository by searching for %[1]syml%[1]s files + # within the "templates" directory and its subdirectories. + + # Components can be defined; + + # - In single files ending in %[1]s.yml%[1]s for each component, like %[1]stemplates/secret-detection.yml%[1]s. + # - In sub-directories containing %[1]stemplate.yml%[1]s files as entry points, + # for components that bundle together multiple related files. For example, + # %[1]stemplates/secret-detection/template.yml%[1]s. + $ glab release create v1.0.1 --publish-to-catalog +`, "`"), RunE: func(cmd *cobra.Command, args []string) error { var err error opts.RepoOverride, _ = cmd.Flags().GetString("repo") @@ -162,6 +178,7 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { cmd.Flags().StringVarP(&opts.ReleasedAt, "released-at", "D", "", "The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z).") cmd.Flags().StringSliceVarP(&opts.Milestone, "milestone", "m", []string{}, "The title of each milestone the release is associated with.") cmd.Flags().StringVarP(&opts.AssetLinksAsJson, "assets-links", "a", "", "'JSON' string representation of assets links, like `--assets-links='[{\"name\": \"Asset1\", \"url\":\"https:///some/location/1\", \"link_type\": \"other\", \"direct_asset_path\": \"path/to/file\"}]'.`") + cmd.Flags().BoolVar(&opts.PublishToCatalog, "publish-to-catalog", false, "Publish the release to the GitLab CI/CD catalog.") return cmd } @@ -396,6 +413,14 @@ func createRun(opts *CreateOpts) error { } } opts.IO.Logf(color.Bold("%s Release succeeded after %0.2fs.\n"), color.GreenCheck(), time.Since(start).Seconds()) + + if opts.PublishToCatalog { + err = publishtocatalog.Run(opts.IO, client, repo.FullName(), release.TagName) + if err != nil { + return cmdutils.WrapError(err, "failed to publish the release to the GitLab CI/CD catalog") + } + } + return nil } diff --git a/commands/release/create/create_test.go b/commands/release/create/create_test.go index 8a38958c2..ede808aa1 100644 --- a/commands/release/create/create_test.go +++ b/commands/release/create/create_test.go @@ -3,9 +3,12 @@ package create import ( "io" "net/http" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/cli/pkg/httpmock" "gitlab.com/gitlab-org/cli/commands/cmdtest" @@ -307,3 +310,135 @@ func TestReleaseCreate_WithAssetsLinksJSON(t *testing.T) { }) } } + +func TestReleaseCreateWithPublishToCatalog(t *testing.T) { + tests := []struct { + name string + cli string + + wantOutput string + wantBody string + wantErr bool + errMsg string + ciJobToken string + }{ + { + name: "with version", + cli: "0.0.1 --publish-to-catalog", + ciJobToken: "token-123", + wantBody: `{ + "version": "0.0.1", + "metadata": { + "components": [ + { + "component_type": "template", + "name": "component-1", + "spec": { + "inputs": { + "compiler": { + "default": "gcc" + } + } + } + }, + { + "component_type": "template", + "name": "component-2", + "spec": null + }, + { + "component_type": "template", + "name": "component-3", + "spec": { + "inputs": { + "test_framework": { + "default": "unittest" + } + } + } + } + ] + } + }`, + wantOutput: `• Publishing release tag=0.0.1 to the GitLab CI/CD catalog for repo=OWNER/REPO... +✓ Release published: url=https://gitlab.example.com/explore/catalog/my-namespace/my-component-project`, + }, + { + name: "missing CI_JOB_TOKEN", + cli: "0.0.1 --publish-to-catalog", + ciJobToken: "", + wantBody: "", + wantErr: true, + errMsg: "`CI_JOB_TOKEN` environment variable not found", + }, + } + + originalWd, err := os.Getwd() + require.NoError(t, err) + + err = os.Chdir(filepath.Join(originalWd, "publish_to_catalog", "testdata", "test-repo")) + require.NoError(t, err) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakeHTTP := &httpmock.Mocker{ + MatchURL: httpmock.PathAndQuerystring, + } + defer fakeHTTP.Verify(t) + + fakeHTTP.RegisterResponder(http.MethodGet, "/api/v4/projects/OWNER/REPO/releases/0%2E0%2E1", + httpmock.NewStringResponse(http.StatusNotFound, `{"message":"404 Not Found"}`)) + + fakeHTTP.RegisterResponder(http.MethodPost, "/api/v4/projects/OWNER/REPO/releases", + func(req *http.Request) (*http.Response, error) { + resp, _ := httpmock.NewStringResponse(http.StatusCreated, + `{ + "name": "test_release", + "tag_name": "0.0.1", + "description": "bugfix release", + "created_at": "2023-01-19T02:58:32.622Z", + "released_at": "2023-01-19T02:58:32.622Z", + "upcoming_release": false, + "tag_path": "/OWNER/REPO/-/tags/0.0.1", + "_links": { + "self": "https://gitlab.com/OWNER/REPO/-/releases/0.0.1" + } + }`)(req) + return resp, nil + }, + ) + + t.Setenv("CI_JOB_TOKEN", tc.ciJobToken) + + if tc.wantBody != "" { + fakeHTTP.RegisterResponder(http.MethodPost, "/api/v4/projects/OWNER/REPO/catalog/publish", + func(req *http.Request) (*http.Response, error) { + body, _ := io.ReadAll(req.Body) + + assert.JSONEq(t, tc.wantBody, string(body)) + assert.Equal(t, "token-123", req.Header.Get("JOB-TOKEN")) + + response := httpmock.NewJSONResponse(http.StatusOK, map[string]interface{}{ + "catalog_url": "https://gitlab.example.com/explore/catalog/my-namespace/my-component-project", + }) + + return response(req) + }, + ) + } + + output, err := runCommand(fakeHTTP, false, tc.cli) + + if tc.wantErr { + assert.Error(t, err) + assert.Equal(t, tc.errMsg, err.Error()) + } else { + assert.NoError(t, err) + assert.Contains(t, output.Stderr(), tc.wantOutput) + } + }) + } + + err = os.Chdir(originalWd) + require.NoError(t, err) +} diff --git a/commands/release/create/publish_to_catalog/run.go b/commands/release/create/publish_to_catalog/run.go new file mode 100644 index 000000000..9887c1fbd --- /dev/null +++ b/commands/release/create/publish_to_catalog/run.go @@ -0,0 +1,181 @@ +package publishtocatalog + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + + "github.com/xanzy/go-gitlab" + "gitlab.com/gitlab-org/cli/commands/cmdutils" + "gitlab.com/gitlab-org/cli/pkg/iostreams" + "gopkg.in/yaml.v3" +) + +const ( + publishToCatalogApiPath = "projects/%s/catalog/publish" + jobTokenVarName = "CI_JOB_TOKEN" + jobTokenHeaderName = "JOB-TOKEN" + templatesDirName = "templates" +) + +type publishToCatalogRequest struct { + Version string `json:"version"` + Metadata map[string]interface{} `json:"metadata"` +} + +type publishToCatalogResponse struct { + CatalogUrl string `json:"catalog_url"` +} + +func Run(io *iostreams.IOStreams, client *gitlab.Client, repoName string, tagName string) error { + color := io.Color() + + io.Logf("%s Publishing release %s=%s to the GitLab CI/CD catalog for %s=%s...\n", + color.ProgressIcon(), + color.Blue("tag"), tagName, + color.Blue("repo"), repoName) + + ciJobToken := os.Getenv(jobTokenVarName) + if ciJobToken == "" { + return fmt.Errorf("`%s` environment variable not found", jobTokenVarName) + } + + body, err := publishToCatalogRequestBody(tagName) + if err != nil { + return cmdutils.WrapError(err, "failed to create a request body") + } + + path := fmt.Sprintf(publishToCatalogApiPath, url.PathEscape(repoName)) + request, err := client.NewRequest(http.MethodPost, path, body, nil) + if err != nil { + return cmdutils.WrapError(err, "failed to create a request.") + } + + request.Header.Set(jobTokenHeaderName, ciJobToken) + + var response publishToCatalogResponse + _, err = client.Do(request, &response) + if err != nil { + return err + } + + io.Logf("%s Release published: %s=%s\n", color.GreenCheck(), + color.Blue("url"), response.CatalogUrl) + + return nil +} + +func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, error) { + components, err := fetchTemplates() + if err != nil { + return nil, cmdutils.WrapError(err, "failed to fetch components") + } + + metadata := make(map[string]interface{}) + componentsData := make([]map[string]interface{}, 0, len(components)) + + for name, path := range components { + spec, err := extractSpec(path) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to extract spec") + } + + componentsData = append(componentsData, map[string]interface{}{ + "name": name, + "spec": spec, + "component_type": "template", + }) + } + + metadata["components"] = componentsData + + return &publishToCatalogRequest{ + Version: version, + Metadata: metadata, + }, nil +} + +func fetchTemplates() (map[string]string, error) { + baseDir, err := os.Getwd() + if err != nil { + return nil, cmdutils.WrapError(err, "failed to get working directory") + } + + templates := make(map[string]string) + + paths, err := fetchTemplatePaths(baseDir) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to fetch template paths") + } + + for _, path := range paths { + componentName, err := extractComponentName(baseDir, path) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to extract component name") + } + + templates[componentName] = path + } + + return templates, nil +} + +func fetchTemplatePaths(baseDir string) ([]string, error) { + templatesDir := filepath.Join(baseDir, templatesDirName) + + var yamlFiles []string + + err := filepath.WalkDir(templatesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return cmdutils.WrapError(err, "failed to walk directory") + } + + if filepath.Ext(d.Name()) == ".yml" { + yamlFiles = append(yamlFiles, path) + } + + return nil + }) + if err != nil { + return nil, err + } + + return yamlFiles, nil +} + +func extractComponentName(baseDir string, path string) (string, error) { + relativePath := path[len(baseDir)+1:] + + dirname := filepath.Dir(relativePath) + fileExt := filepath.Ext(relativePath) + filename := filepath.Base(relativePath) + filenameWithoutExt := filename[:len(filename)-len(fileExt)] + + if dirname == templatesDirName { + return filenameWithoutExt, nil + } + + return filepath.Base(dirname), nil +} + +type specDef struct { + Spec map[string]interface{} `yaml:"spec"` +} + +func extractSpec(componentPath string) (map[string]interface{}, error) { + content, err := os.ReadFile(componentPath) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to read file") + } + + var spec specDef + + err = yaml.Unmarshal(content, &spec) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to unmarshal YAML") + } + + return spec.Spec, nil +} diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml new file mode 100644 index 000000000..d1105a72c --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml @@ -0,0 +1,10 @@ +spec: + inputs: + compiler: + default: gcc + +--- + +test: + script: + - echo $[[ inputs.compiler ]] diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml new file mode 100644 index 000000000..ab84ce56f --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml @@ -0,0 +1,3 @@ +test: + script: + - echo no-spec-inputs diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml new file mode 100644 index 000000000..107f55bee --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml @@ -0,0 +1,10 @@ +spec: + inputs: + test_framework: + default: unittest + +--- + +test: + script: + - echo $[[ inputs.test_framework ]] diff --git a/docs/source/release/create.md b/docs/source/release/create.md index 3a53d8d54..1a5a760f8 100644 --- a/docs/source/release/create.md +++ b/docs/source/release/create.md @@ -68,6 +68,20 @@ $ glab release create v1.0.1 --assets-links=' } ]' +# Create a release and publish it to the GitLab CI/CD catalog +# This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. +# The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. +# This command retrieves components from the current repository by searching for `yml` files +# within the "templates" directory and its subdirectories. + +# Components can be defined; + +# - In single files ending in `.yml` for each component, like `templates/secret-detection.yml`. +# - In sub-directories containing `template.yml` files as entry points, +# for components that bundle together multiple related files. For example, +# `templates/secret-detection/template.yml`. +$ glab release create v1.0.1 --publish-to-catalog + ``` ## Options @@ -78,6 +92,7 @@ $ glab release create v1.0.1 --assets-links=' -n, --name string The release name or title. -N, --notes string The release notes or description. You can use Markdown. -F, --notes-file string Read release notes 'file'. Specify '-' as the value to read from stdin. + --publish-to-catalog Publish the release to the GitLab CI/CD catalog. -r, --ref string If the specified tag doesn't exist, the release is created from ref and tagged with the specified tag name. It can be a commit SHA, another tag name, or a branch name. -D, --released-at string The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z). -T, --tag-message string Message to use if creating a new annotated tag. diff --git a/docs/source/release/publish-to-catalog.md b/docs/source/release/publish-to-catalog.md new file mode 100644 index 000000000..8e2babfde --- /dev/null +++ b/docs/source/release/publish-to-catalog.md @@ -0,0 +1,54 @@ +--- +stage: Create +group: Code Review +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + + + +# `glab release publish-to-catalog` + +Publish a release to the GitLab CI/CD catalog + +## Synopsis + +Publish a release to the GitLab CI/CD catalog. + +Publish a release to the GitLab CI/CD catalog. +The release must be created before it can be published. +This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. +To prevent running this manually, there are a few restrictions: + +- The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. +- The current user must be the same as the one who created the release. + +This command retrieves components from the current repository by searching for `yml` files +within the "templates" directory and its subdirectories. + +Components can be defined; + +- In single files ending in `.yml` for each component, like `templates/secret-detection.yml`. +- In sub-directories containing `template.yml` files as entry points, + for components that bundle together multiple related files. For example, + `templates/secret-detection/template.yml`. + +```plaintext +glab release publish-to-catalog [flags] +``` + +## Examples + +```plaintext +$ glab release publish-to-catalog v1.0.0 + +``` + +## Options inherited from parent commands + +```plaintext + --help Show help for this command. + -R, --repo OWNER/REPO Select another repository. Can use either OWNER/REPO or `GROUP/NAMESPACE/REPO` format. Also accepts full URL or Git URL. +``` -- GitLab From d63ff2f105a692b932c9aa69169a2da5359d8719 Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Thu, 21 Nov 2024 10:01:38 +0100 Subject: [PATCH 2/6] Refactoring and new tests --- .../release/create/publish_to_catalog/run.go | 45 ++++++++--- .../create/publish_to_catalog/run_test.go | 74 +++++++++++++++++++ .../abc_templates/not_a_component.yml | 10 +++ .../not_a_component2/template.yml | 10 +++ .../component-3/another_helper_file.yml | 9 +++ .../contains-no-template/just_a_file.yml | 2 + 6 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 commands/release/create/publish_to_catalog/run_test.go create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml create mode 100644 commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml diff --git a/commands/release/create/publish_to_catalog/run.go b/commands/release/create/publish_to_catalog/run.go index 9887c1fbd..e8cfa64b3 100644 --- a/commands/release/create/publish_to_catalog/run.go +++ b/commands/release/create/publish_to_catalog/run.go @@ -18,6 +18,8 @@ const ( jobTokenVarName = "CI_JOB_TOKEN" jobTokenHeaderName = "JOB-TOKEN" templatesDirName = "templates" + templateFileExt = ".yml" + templateFileName = "template.yml" ) type publishToCatalogRequest struct { @@ -68,7 +70,12 @@ func Run(io *iostreams.IOStreams, client *gitlab.Client, repoName string, tagNam } func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, error) { - components, err := fetchTemplates() + baseDir, err := os.Getwd() + if err != nil { + return nil, cmdutils.WrapError(err, "failed to get working directory") + } + + components, err := fetchTemplates(baseDir) if err != nil { return nil, cmdutils.WrapError(err, "failed to fetch components") } @@ -97,12 +104,11 @@ func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, erro }, nil } -func fetchTemplates() (map[string]string, error) { - baseDir, err := os.Getwd() - if err != nil { - return nil, cmdutils.WrapError(err, "failed to get working directory") - } - +// fetchTemplates returns a map of component names to their paths. +// The component name is either the name of the file without the extension in the "templates" directory of the project +// or the name of the directory containing a "template.yml" file in the "templates" directory. +// More information: https://docs.gitlab.com/ee/ci/components/index.html#directory-structure +func fetchTemplates(baseDir string) (map[string]string, error) { templates := make(map[string]string) paths, err := fetchTemplatePaths(baseDir) @@ -116,12 +122,15 @@ func fetchTemplates() (map[string]string, error) { return nil, cmdutils.WrapError(err, "failed to extract component name") } - templates[componentName] = path + if componentName != "" { + templates[componentName] = path + } } return templates, nil } +// fetchTemplatePaths returns a list of the possible component paths to the YAML files in the "templates" directory. func fetchTemplatePaths(baseDir string) ([]string, error) { templatesDir := filepath.Join(baseDir, templatesDirName) @@ -132,7 +141,7 @@ func fetchTemplatePaths(baseDir string) ([]string, error) { return cmdutils.WrapError(err, "failed to walk directory") } - if filepath.Ext(d.Name()) == ".yml" { + if filepath.Ext(d.Name()) == templateFileExt { yamlFiles = append(yamlFiles, path) } @@ -145,25 +154,39 @@ func fetchTemplatePaths(baseDir string) ([]string, error) { return yamlFiles, nil } +// extractComponentName returns the valid component name from the path if it is a valid component path. +// valid component paths: +// 1. All YAML files in the "templates" directory. +// 2. All "template.yml" files in the subdirectories of the "templates" directory. func extractComponentName(baseDir string, path string) (string, error) { - relativePath := path[len(baseDir)+1:] + relativePath, err := filepath.Rel(baseDir, path) + if err != nil { + return "", err + } dirname := filepath.Dir(relativePath) fileExt := filepath.Ext(relativePath) filename := filepath.Base(relativePath) filenameWithoutExt := filename[:len(filename)-len(fileExt)] + // All YAML files in the "templates" directory. if dirname == templatesDirName { return filenameWithoutExt, nil } - return filepath.Base(dirname), nil + // All "template.yml" files in the subdirectories of the "templates" directory. + if filename == templateFileName { + return filepath.Base(dirname), nil + } else { + return "", nil + } } type specDef struct { Spec map[string]interface{} `yaml:"spec"` } +// extractSpec returns the spec from the component file. func extractSpec(componentPath string) (map[string]interface{}, error) { content, err := os.ReadFile(componentPath) if err != nil { diff --git a/commands/release/create/publish_to_catalog/run_test.go b/commands/release/create/publish_to_catalog/run_test.go new file mode 100644 index 000000000..1d9fbb48d --- /dev/null +++ b/commands/release/create/publish_to_catalog/run_test.go @@ -0,0 +1,74 @@ +package publishtocatalog + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fetchTemplates(t *testing.T) { + err := os.Chdir("./testdata/test-repo") + require.NoError(t, err) + t.Cleanup(func() { + err := os.Chdir("../..") + require.NoError(t, err) + }) + + wd, err := os.Getwd() + require.NoError(t, err) + want := map[string]string{ + "component-1": filepath.Join(wd, "templates/component-1.yml"), + "component-2": filepath.Join(wd, "templates/component-2.yml"), + "component-3": filepath.Join(wd, "templates/component-3", "template.yml"), + } + got, err := fetchTemplates(wd) + require.NoError(t, err) + + for k, v := range want { + require.Equal(t, got[k], v) + } +} + +func Test_extractComponentName(t *testing.T) { + err := os.Chdir("./testdata/test-repo") + require.NoError(t, err) + t.Cleanup(func() { + err := os.Chdir("../..") + require.NoError(t, err) + }) + + wd, err := os.Getwd() + require.NoError(t, err) + + tests := []struct { + name string + path string + expected string + }{ + { + name: "valid component path", + path: filepath.Join(wd, "templates/component-1.yml"), + expected: "component-1", + }, + { + name: "valid component path", + path: filepath.Join(wd, "templates/component-2", "template.yml"), + expected: "component-2", + }, + { + name: "invalid component path", + path: filepath.Join(wd, "abc_templates/component-3.yml"), + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractComponentName(wd, tt.path) + require.NoError(t, err) + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml new file mode 100644 index 000000000..92fc6efc4 --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml @@ -0,0 +1,10 @@ +spec: + inputs: + abc: + default: eee + +--- + +test: + script: + - echo a diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml new file mode 100644 index 000000000..046b185d2 --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml @@ -0,0 +1,10 @@ +spec: + inputs: + fff: + default: ggg + +--- + +test: + script: + - echo a diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml new file mode 100644 index 000000000..ee778521d --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml @@ -0,0 +1,9 @@ +spec: + inputs: + should_be_ignored: + default: hello + +--- + +build: + script: echo "Hello, World!" diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml new file mode 100644 index 000000000..1b32e3e42 --- /dev/null +++ b/commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml @@ -0,0 +1,2 @@ +test: + script: echo "Hello, world!" -- GitLab From 643006d43038169d147311c8d94ac2bc0e7ec4f4 Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Mon, 25 Nov 2024 09:04:26 +0100 Subject: [PATCH 3/6] Fix flaky test by sorting --- commands/release/create/publish_to_catalog/run.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commands/release/create/publish_to_catalog/run.go b/commands/release/create/publish_to_catalog/run.go index e8cfa64b3..7c834b1b5 100644 --- a/commands/release/create/publish_to_catalog/run.go +++ b/commands/release/create/publish_to_catalog/run.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "github.com/xanzy/go-gitlab" "gitlab.com/gitlab-org/cli/commands/cmdutils" @@ -96,6 +97,10 @@ func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, erro }) } + sort.Slice(componentsData, func(i, j int) bool { + return componentsData[i]["name"].(string) < componentsData[j]["name"].(string) + }) + metadata["components"] = componentsData return &publishToCatalogRequest{ -- GitLab From 5b3a7f460d9c01e1f6a0c4c9d1a9f4713cc40e02 Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Mon, 25 Nov 2024 10:39:12 +0100 Subject: [PATCH 4/6] Rename the package --- .../run.go => catalog/publish.go} | 4 +- .../run_test.go => catalog/publish_test.go} | 2 +- .../abc_templates/not_a_component.yml | 0 .../not_a_component2/template.yml | 0 .../test-repo/templates/component-1.yml | 0 .../test-repo/templates/component-2.yml | 0 .../component-3/another_helper_file.yml | 0 .../templates/component-3/template.yml | 0 .../contains-no-template/just_a_file.yml | 0 commands/release/create/create.go | 4 +- commands/release/create/create_test.go | 2 +- docs/source/release/publish-to-catalog.md | 54 ------------------- 12 files changed, 6 insertions(+), 60 deletions(-) rename commands/release/create/{publish_to_catalog/run.go => catalog/publish.go} (97%) rename commands/release/create/{publish_to_catalog/run_test.go => catalog/publish_test.go} (98%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/abc_templates/not_a_component.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/abc_templates/not_a_component2/template.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/templates/component-1.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/templates/component-2.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/templates/component-3/another_helper_file.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/templates/component-3/template.yml (100%) rename commands/release/create/{publish_to_catalog => catalog}/testdata/test-repo/templates/contains-no-template/just_a_file.yml (100%) delete mode 100644 docs/source/release/publish-to-catalog.md diff --git a/commands/release/create/publish_to_catalog/run.go b/commands/release/create/catalog/publish.go similarity index 97% rename from commands/release/create/publish_to_catalog/run.go rename to commands/release/create/catalog/publish.go index 7c834b1b5..6ea2a4cae 100644 --- a/commands/release/create/publish_to_catalog/run.go +++ b/commands/release/create/catalog/publish.go @@ -1,4 +1,4 @@ -package publishtocatalog +package catalog import ( "fmt" @@ -32,7 +32,7 @@ type publishToCatalogResponse struct { CatalogUrl string `json:"catalog_url"` } -func Run(io *iostreams.IOStreams, client *gitlab.Client, repoName string, tagName string) error { +func Publish(io *iostreams.IOStreams, client *gitlab.Client, repoName string, tagName string) error { color := io.Color() io.Logf("%s Publishing release %s=%s to the GitLab CI/CD catalog for %s=%s...\n", diff --git a/commands/release/create/publish_to_catalog/run_test.go b/commands/release/create/catalog/publish_test.go similarity index 98% rename from commands/release/create/publish_to_catalog/run_test.go rename to commands/release/create/catalog/publish_test.go index 1d9fbb48d..7e0be50c3 100644 --- a/commands/release/create/publish_to_catalog/run_test.go +++ b/commands/release/create/catalog/publish_test.go @@ -1,4 +1,4 @@ -package publishtocatalog +package catalog import ( "os" diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component.yml rename to commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml rename to commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-1.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-1.yml rename to commands/release/create/catalog/testdata/test-repo/templates/component-1.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-2.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-2.yml rename to commands/release/create/catalog/testdata/test-repo/templates/component-2.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-3/another_helper_file.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/another_helper_file.yml rename to commands/release/create/catalog/testdata/test-repo/templates/component-3/another_helper_file.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-3/template.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/templates/component-3/template.yml rename to commands/release/create/catalog/testdata/test-repo/templates/component-3/template.yml diff --git a/commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml b/commands/release/create/catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml similarity index 100% rename from commands/release/create/publish_to_catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml rename to commands/release/create/catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml diff --git a/commands/release/create/create.go b/commands/release/create/create.go index 7767c4834..58568fdda 100644 --- a/commands/release/create/create.go +++ b/commands/release/create/create.go @@ -10,7 +10,7 @@ import ( "strings" "time" - publishtocatalog "gitlab.com/gitlab-org/cli/commands/release/create/publish_to_catalog" + catalog "gitlab.com/gitlab-org/cli/commands/release/create/catalog" "gitlab.com/gitlab-org/cli/commands/release/releaseutils" "gitlab.com/gitlab-org/cli/commands/release/releaseutils/upload" @@ -415,7 +415,7 @@ func createRun(opts *CreateOpts) error { opts.IO.Logf(color.Bold("%s Release succeeded after %0.2fs.\n"), color.GreenCheck(), time.Since(start).Seconds()) if opts.PublishToCatalog { - err = publishtocatalog.Run(opts.IO, client, repo.FullName(), release.TagName) + err = catalog.Publish(opts.IO, client, repo.FullName(), release.TagName) if err != nil { return cmdutils.WrapError(err, "failed to publish the release to the GitLab CI/CD catalog") } diff --git a/commands/release/create/create_test.go b/commands/release/create/create_test.go index ede808aa1..2d11666af 100644 --- a/commands/release/create/create_test.go +++ b/commands/release/create/create_test.go @@ -376,7 +376,7 @@ func TestReleaseCreateWithPublishToCatalog(t *testing.T) { originalWd, err := os.Getwd() require.NoError(t, err) - err = os.Chdir(filepath.Join(originalWd, "publish_to_catalog", "testdata", "test-repo")) + err = os.Chdir(filepath.Join(originalWd, "catalog", "testdata", "test-repo")) require.NoError(t, err) for _, tc := range tests { diff --git a/docs/source/release/publish-to-catalog.md b/docs/source/release/publish-to-catalog.md deleted file mode 100644 index 8e2babfde..000000000 --- a/docs/source/release/publish-to-catalog.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -stage: Create -group: Code Review -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments ---- - - - -# `glab release publish-to-catalog` - -Publish a release to the GitLab CI/CD catalog - -## Synopsis - -Publish a release to the GitLab CI/CD catalog. - -Publish a release to the GitLab CI/CD catalog. -The release must be created before it can be published. -This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. -To prevent running this manually, there are a few restrictions: - -- The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. -- The current user must be the same as the one who created the release. - -This command retrieves components from the current repository by searching for `yml` files -within the "templates" directory and its subdirectories. - -Components can be defined; - -- In single files ending in `.yml` for each component, like `templates/secret-detection.yml`. -- In sub-directories containing `template.yml` files as entry points, - for components that bundle together multiple related files. For example, - `templates/secret-detection/template.yml`. - -```plaintext -glab release publish-to-catalog [flags] -``` - -## Examples - -```plaintext -$ glab release publish-to-catalog v1.0.0 - -``` - -## Options inherited from parent commands - -```plaintext - --help Show help for this command. - -R, --repo OWNER/REPO Select another repository. Can use either OWNER/REPO or `GROUP/NAMESPACE/REPO` format. Also accepts full URL or Git URL. -``` -- GitLab From d2d9ca84d67398caadd0307f76a21f2ac670ab78 Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Tue, 26 Nov 2024 09:53:32 +0100 Subject: [PATCH 5/6] Remove the CI_JOB_TOKEN requirement --- commands/release/create/catalog/publish.go | 9 --------- commands/release/create/create_test.go | 17 ++--------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/commands/release/create/catalog/publish.go b/commands/release/create/catalog/publish.go index 6ea2a4cae..a14fece4b 100644 --- a/commands/release/create/catalog/publish.go +++ b/commands/release/create/catalog/publish.go @@ -16,8 +16,6 @@ import ( const ( publishToCatalogApiPath = "projects/%s/catalog/publish" - jobTokenVarName = "CI_JOB_TOKEN" - jobTokenHeaderName = "JOB-TOKEN" templatesDirName = "templates" templateFileExt = ".yml" templateFileName = "template.yml" @@ -40,11 +38,6 @@ func Publish(io *iostreams.IOStreams, client *gitlab.Client, repoName string, ta color.Blue("tag"), tagName, color.Blue("repo"), repoName) - ciJobToken := os.Getenv(jobTokenVarName) - if ciJobToken == "" { - return fmt.Errorf("`%s` environment variable not found", jobTokenVarName) - } - body, err := publishToCatalogRequestBody(tagName) if err != nil { return cmdutils.WrapError(err, "failed to create a request body") @@ -56,8 +49,6 @@ func Publish(io *iostreams.IOStreams, client *gitlab.Client, repoName string, ta return cmdutils.WrapError(err, "failed to create a request.") } - request.Header.Set(jobTokenHeaderName, ciJobToken) - var response publishToCatalogResponse _, err = client.Do(request, &response) if err != nil { diff --git a/commands/release/create/create_test.go b/commands/release/create/create_test.go index 2d11666af..f6d3e535a 100644 --- a/commands/release/create/create_test.go +++ b/commands/release/create/create_test.go @@ -320,12 +320,10 @@ func TestReleaseCreateWithPublishToCatalog(t *testing.T) { wantBody string wantErr bool errMsg string - ciJobToken string }{ { - name: "with version", - cli: "0.0.1 --publish-to-catalog", - ciJobToken: "token-123", + name: "with version", + cli: "0.0.1 --publish-to-catalog", wantBody: `{ "version": "0.0.1", "metadata": { @@ -363,14 +361,6 @@ func TestReleaseCreateWithPublishToCatalog(t *testing.T) { wantOutput: `• Publishing release tag=0.0.1 to the GitLab CI/CD catalog for repo=OWNER/REPO... ✓ Release published: url=https://gitlab.example.com/explore/catalog/my-namespace/my-component-project`, }, - { - name: "missing CI_JOB_TOKEN", - cli: "0.0.1 --publish-to-catalog", - ciJobToken: "", - wantBody: "", - wantErr: true, - errMsg: "`CI_JOB_TOKEN` environment variable not found", - }, } originalWd, err := os.Getwd() @@ -408,15 +398,12 @@ func TestReleaseCreateWithPublishToCatalog(t *testing.T) { }, ) - t.Setenv("CI_JOB_TOKEN", tc.ciJobToken) - if tc.wantBody != "" { fakeHTTP.RegisterResponder(http.MethodPost, "/api/v4/projects/OWNER/REPO/catalog/publish", func(req *http.Request) (*http.Response, error) { body, _ := io.ReadAll(req.Body) assert.JSONEq(t, tc.wantBody, string(body)) - assert.Equal(t, "token-123", req.Header.Get("JOB-TOKEN")) response := httpmock.NewJSONResponse(http.StatusOK, map[string]interface{}{ "catalog_url": "https://gitlab.example.com/explore/catalog/my-namespace/my-component-project", -- GitLab From bd8f75a7503d60a593acab1310984691635621ef Mon Sep 17 00:00:00 2001 From: Furkan Ayhan Date: Tue, 26 Nov 2024 09:53:44 +0100 Subject: [PATCH 6/6] Add EXPERIMENTAL to the flag --- commands/release/create/create.go | 8 +++++--- docs/source/release/create.md | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/commands/release/create/create.go b/commands/release/create/create.go index 58568fdda..8f9d0e5bf 100644 --- a/commands/release/create/create.go +++ b/commands/release/create/create.go @@ -113,11 +113,13 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { } ]' - # Create a release and publish it to the GitLab CI/CD catalog - # This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. + # [EXPERIMENTAL] Create a release and publish it to the GitLab CI/CD catalog + # This command should NOT be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. # The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. # This command retrieves components from the current repository by searching for %[1]syml%[1]s files # within the "templates" directory and its subdirectories. + # This flag will not work if the feature flag %[1]sci_release_cli_catalog_publish_option%[1]s is not enabled + # for the project in the GitLab instance. # Components can be defined; @@ -178,7 +180,7 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { cmd.Flags().StringVarP(&opts.ReleasedAt, "released-at", "D", "", "The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z).") cmd.Flags().StringSliceVarP(&opts.Milestone, "milestone", "m", []string{}, "The title of each milestone the release is associated with.") cmd.Flags().StringVarP(&opts.AssetLinksAsJson, "assets-links", "a", "", "'JSON' string representation of assets links, like `--assets-links='[{\"name\": \"Asset1\", \"url\":\"https:///some/location/1\", \"link_type\": \"other\", \"direct_asset_path\": \"path/to/file\"}]'.`") - cmd.Flags().BoolVar(&opts.PublishToCatalog, "publish-to-catalog", false, "Publish the release to the GitLab CI/CD catalog.") + cmd.Flags().BoolVar(&opts.PublishToCatalog, "publish-to-catalog", false, "[EXPERIMENTAL] Publish the release to the GitLab CI/CD catalog.") return cmd } diff --git a/docs/source/release/create.md b/docs/source/release/create.md index 1a5a760f8..8339d1595 100644 --- a/docs/source/release/create.md +++ b/docs/source/release/create.md @@ -68,11 +68,13 @@ $ glab release create v1.0.1 --assets-links=' } ]' -# Create a release and publish it to the GitLab CI/CD catalog -# This command should not be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. +# [EXPERIMENTAL] Create a release and publish it to the GitLab CI/CD catalog +# This command should NOT be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. # The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. # This command retrieves components from the current repository by searching for `yml` files # within the "templates" directory and its subdirectories. +# This flag will not work if the feature flag `ci_release_cli_catalog_publish_option` is not enabled +# for the project in the GitLab instance. # Components can be defined; @@ -92,7 +94,7 @@ $ glab release create v1.0.1 --publish-to-catalog -n, --name string The release name or title. -N, --notes string The release notes or description. You can use Markdown. -F, --notes-file string Read release notes 'file'. Specify '-' as the value to read from stdin. - --publish-to-catalog Publish the release to the GitLab CI/CD catalog. + --publish-to-catalog [EXPERIMENTAL] Publish the release to the GitLab CI/CD catalog. -r, --ref string If the specified tag doesn't exist, the release is created from ref and tagged with the specified tag name. It can be a commit SHA, another tag name, or a branch name. -D, --released-at string The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z). -T, --tag-message string Message to use if creating a new annotated tag. -- GitLab