diff --git a/commands/release/create/catalog/publish.go b/commands/release/create/catalog/publish.go new file mode 100644 index 0000000000000000000000000000000000000000..a14fece4bce9acb5d672ef787e3f70c2314abe9c --- /dev/null +++ b/commands/release/create/catalog/publish.go @@ -0,0 +1,200 @@ +package catalog + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + + "github.com/xanzy/go-gitlab" + "gitlab.com/gitlab-org/cli/commands/cmdutils" + "gitlab.com/gitlab-org/cli/pkg/iostreams" + "gopkg.in/yaml.v3" +) + +const ( + publishToCatalogApiPath = "projects/%s/catalog/publish" + templatesDirName = "templates" + templateFileExt = ".yml" + templateFileName = "template.yml" +) + +type publishToCatalogRequest struct { + Version string `json:"version"` + Metadata map[string]interface{} `json:"metadata"` +} + +type publishToCatalogResponse struct { + CatalogUrl string `json:"catalog_url"` +} + +func Publish(io *iostreams.IOStreams, client *gitlab.Client, repoName string, tagName string) error { + color := io.Color() + + io.Logf("%s Publishing release %s=%s to the GitLab CI/CD catalog for %s=%s...\n", + color.ProgressIcon(), + color.Blue("tag"), tagName, + color.Blue("repo"), repoName) + + body, err := publishToCatalogRequestBody(tagName) + if err != nil { + return cmdutils.WrapError(err, "failed to create a request body") + } + + path := fmt.Sprintf(publishToCatalogApiPath, url.PathEscape(repoName)) + request, err := client.NewRequest(http.MethodPost, path, body, nil) + if err != nil { + return cmdutils.WrapError(err, "failed to create a request.") + } + + var response publishToCatalogResponse + _, err = client.Do(request, &response) + if err != nil { + return err + } + + io.Logf("%s Release published: %s=%s\n", color.GreenCheck(), + color.Blue("url"), response.CatalogUrl) + + return nil +} + +func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, error) { + baseDir, err := os.Getwd() + if err != nil { + return nil, cmdutils.WrapError(err, "failed to get working directory") + } + + components, err := fetchTemplates(baseDir) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to fetch components") + } + + metadata := make(map[string]interface{}) + componentsData := make([]map[string]interface{}, 0, len(components)) + + for name, path := range components { + spec, err := extractSpec(path) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to extract spec") + } + + componentsData = append(componentsData, map[string]interface{}{ + "name": name, + "spec": spec, + "component_type": "template", + }) + } + + sort.Slice(componentsData, func(i, j int) bool { + return componentsData[i]["name"].(string) < componentsData[j]["name"].(string) + }) + + metadata["components"] = componentsData + + return &publishToCatalogRequest{ + Version: version, + Metadata: metadata, + }, nil +} + +// fetchTemplates returns a map of component names to their paths. +// The component name is either the name of the file without the extension in the "templates" directory of the project +// or the name of the directory containing a "template.yml" file in the "templates" directory. +// More information: https://docs.gitlab.com/ee/ci/components/index.html#directory-structure +func fetchTemplates(baseDir string) (map[string]string, error) { + templates := make(map[string]string) + + paths, err := fetchTemplatePaths(baseDir) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to fetch template paths") + } + + for _, path := range paths { + componentName, err := extractComponentName(baseDir, path) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to extract component name") + } + + if componentName != "" { + templates[componentName] = path + } + } + + return templates, nil +} + +// fetchTemplatePaths returns a list of the possible component paths to the YAML files in the "templates" directory. +func fetchTemplatePaths(baseDir string) ([]string, error) { + templatesDir := filepath.Join(baseDir, templatesDirName) + + var yamlFiles []string + + err := filepath.WalkDir(templatesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return cmdutils.WrapError(err, "failed to walk directory") + } + + if filepath.Ext(d.Name()) == templateFileExt { + yamlFiles = append(yamlFiles, path) + } + + return nil + }) + if err != nil { + return nil, err + } + + return yamlFiles, nil +} + +// extractComponentName returns the valid component name from the path if it is a valid component path. +// valid component paths: +// 1. All YAML files in the "templates" directory. +// 2. All "template.yml" files in the subdirectories of the "templates" directory. +func extractComponentName(baseDir string, path string) (string, error) { + relativePath, err := filepath.Rel(baseDir, path) + if err != nil { + return "", err + } + + dirname := filepath.Dir(relativePath) + fileExt := filepath.Ext(relativePath) + filename := filepath.Base(relativePath) + filenameWithoutExt := filename[:len(filename)-len(fileExt)] + + // All YAML files in the "templates" directory. + if dirname == templatesDirName { + return filenameWithoutExt, nil + } + + // All "template.yml" files in the subdirectories of the "templates" directory. + if filename == templateFileName { + return filepath.Base(dirname), nil + } else { + return "", nil + } +} + +type specDef struct { + Spec map[string]interface{} `yaml:"spec"` +} + +// extractSpec returns the spec from the component file. +func extractSpec(componentPath string) (map[string]interface{}, error) { + content, err := os.ReadFile(componentPath) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to read file") + } + + var spec specDef + + err = yaml.Unmarshal(content, &spec) + if err != nil { + return nil, cmdutils.WrapError(err, "failed to unmarshal YAML") + } + + return spec.Spec, nil +} diff --git a/commands/release/create/catalog/publish_test.go b/commands/release/create/catalog/publish_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7e0be50c3529fc7495f13a1d480d6b89ea58a0ad --- /dev/null +++ b/commands/release/create/catalog/publish_test.go @@ -0,0 +1,74 @@ +package catalog + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fetchTemplates(t *testing.T) { + err := os.Chdir("./testdata/test-repo") + require.NoError(t, err) + t.Cleanup(func() { + err := os.Chdir("../..") + require.NoError(t, err) + }) + + wd, err := os.Getwd() + require.NoError(t, err) + want := map[string]string{ + "component-1": filepath.Join(wd, "templates/component-1.yml"), + "component-2": filepath.Join(wd, "templates/component-2.yml"), + "component-3": filepath.Join(wd, "templates/component-3", "template.yml"), + } + got, err := fetchTemplates(wd) + require.NoError(t, err) + + for k, v := range want { + require.Equal(t, got[k], v) + } +} + +func Test_extractComponentName(t *testing.T) { + err := os.Chdir("./testdata/test-repo") + require.NoError(t, err) + t.Cleanup(func() { + err := os.Chdir("../..") + require.NoError(t, err) + }) + + wd, err := os.Getwd() + require.NoError(t, err) + + tests := []struct { + name string + path string + expected string + }{ + { + name: "valid component path", + path: filepath.Join(wd, "templates/component-1.yml"), + expected: "component-1", + }, + { + name: "valid component path", + path: filepath.Join(wd, "templates/component-2", "template.yml"), + expected: "component-2", + }, + { + name: "invalid component path", + path: filepath.Join(wd, "abc_templates/component-3.yml"), + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractComponentName(wd, tt.path) + require.NoError(t, err) + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component.yml b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component.yml new file mode 100644 index 0000000000000000000000000000000000000000..92fc6efc451c52b7d41ac78db7decde444de7faf --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component.yml @@ -0,0 +1,10 @@ +spec: + inputs: + abc: + default: eee + +--- + +test: + script: + - echo a diff --git a/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml new file mode 100644 index 0000000000000000000000000000000000000000..046b185d2d80a374a5f8a65125cfba33cd639fee --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/abc_templates/not_a_component2/template.yml @@ -0,0 +1,10 @@ +spec: + inputs: + fff: + default: ggg + +--- + +test: + script: + - echo a diff --git a/commands/release/create/catalog/testdata/test-repo/templates/component-1.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-1.yml new file mode 100644 index 0000000000000000000000000000000000000000..d1105a72c832c00a883d80d0c304aaef82044935 --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/templates/component-1.yml @@ -0,0 +1,10 @@ +spec: + inputs: + compiler: + default: gcc + +--- + +test: + script: + - echo $[[ inputs.compiler ]] diff --git a/commands/release/create/catalog/testdata/test-repo/templates/component-2.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-2.yml new file mode 100644 index 0000000000000000000000000000000000000000..ab84ce56f61c0b323b4f59cb725c751b469a971e --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/templates/component-2.yml @@ -0,0 +1,3 @@ +test: + script: + - echo no-spec-inputs diff --git a/commands/release/create/catalog/testdata/test-repo/templates/component-3/another_helper_file.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-3/another_helper_file.yml new file mode 100644 index 0000000000000000000000000000000000000000..ee778521d5b81e15db1a746f718b79b3208dfa1c --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/templates/component-3/another_helper_file.yml @@ -0,0 +1,9 @@ +spec: + inputs: + should_be_ignored: + default: hello + +--- + +build: + script: echo "Hello, World!" diff --git a/commands/release/create/catalog/testdata/test-repo/templates/component-3/template.yml b/commands/release/create/catalog/testdata/test-repo/templates/component-3/template.yml new file mode 100644 index 0000000000000000000000000000000000000000..107f55bee618127da61c10be762126067e84fee2 --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/templates/component-3/template.yml @@ -0,0 +1,10 @@ +spec: + inputs: + test_framework: + default: unittest + +--- + +test: + script: + - echo $[[ inputs.test_framework ]] diff --git a/commands/release/create/catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml b/commands/release/create/catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml new file mode 100644 index 0000000000000000000000000000000000000000..1b32e3e4212925d097bf0f2413e27f162f88acf3 --- /dev/null +++ b/commands/release/create/catalog/testdata/test-repo/templates/contains-no-template/just_a_file.yml @@ -0,0 +1,2 @@ +test: + script: echo "Hello, world!" diff --git a/commands/release/create/create.go b/commands/release/create/create.go index a291dccb299fe6c16d4bfd3c06fdeb753f9fe845..8f9d0e5bfd583a373d55df30616fd3ee620a3251 100644 --- a/commands/release/create/create.go +++ b/commands/release/create/create.go @@ -10,6 +10,7 @@ import ( "strings" "time" + catalog "gitlab.com/gitlab-org/cli/commands/release/create/catalog" "gitlab.com/gitlab-org/cli/commands/release/releaseutils" "gitlab.com/gitlab-org/cli/commands/release/releaseutils/upload" @@ -41,6 +42,7 @@ type CreateOpts struct { AssetLinksAsJson string ReleasedAt string RepoOverride string + PublishToCatalog bool NoteProvided bool ReleaseNotesAction string @@ -78,7 +80,7 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { To fetch the new tag locally after the release, run %[1]sgit fetch --tags origin%[1]s. `, "`"), Args: cmdutils.MinimumArgs(1, "no tag name provided"), - Example: heredoc.Doc(` + Example: heredoc.Docf(` # Interactively create a release $ glab release create v1.0.1 @@ -110,7 +112,23 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { "direct_asset_path": "path/to/file" } ]' -`), + + # [EXPERIMENTAL] Create a release and publish it to the GitLab CI/CD catalog + # This command should NOT be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. + # The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. + # This command retrieves components from the current repository by searching for %[1]syml%[1]s files + # within the "templates" directory and its subdirectories. + # This flag will not work if the feature flag %[1]sci_release_cli_catalog_publish_option%[1]s is not enabled + # for the project in the GitLab instance. + + # Components can be defined; + + # - In single files ending in %[1]s.yml%[1]s for each component, like %[1]stemplates/secret-detection.yml%[1]s. + # - In sub-directories containing %[1]stemplate.yml%[1]s files as entry points, + # for components that bundle together multiple related files. For example, + # %[1]stemplates/secret-detection/template.yml%[1]s. + $ glab release create v1.0.1 --publish-to-catalog +`, "`"), RunE: func(cmd *cobra.Command, args []string) error { var err error opts.RepoOverride, _ = cmd.Flags().GetString("repo") @@ -162,6 +180,7 @@ func NewCmdCreate(f *cmdutils.Factory) *cobra.Command { cmd.Flags().StringVarP(&opts.ReleasedAt, "released-at", "D", "", "The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z).") cmd.Flags().StringSliceVarP(&opts.Milestone, "milestone", "m", []string{}, "The title of each milestone the release is associated with.") cmd.Flags().StringVarP(&opts.AssetLinksAsJson, "assets-links", "a", "", "'JSON' string representation of assets links, like `--assets-links='[{\"name\": \"Asset1\", \"url\":\"https:///some/location/1\", \"link_type\": \"other\", \"direct_asset_path\": \"path/to/file\"}]'.`") + cmd.Flags().BoolVar(&opts.PublishToCatalog, "publish-to-catalog", false, "[EXPERIMENTAL] Publish the release to the GitLab CI/CD catalog.") return cmd } @@ -396,6 +415,14 @@ func createRun(opts *CreateOpts) error { } } opts.IO.Logf(color.Bold("%s Release succeeded after %0.2fs.\n"), color.GreenCheck(), time.Since(start).Seconds()) + + if opts.PublishToCatalog { + err = catalog.Publish(opts.IO, client, repo.FullName(), release.TagName) + if err != nil { + return cmdutils.WrapError(err, "failed to publish the release to the GitLab CI/CD catalog") + } + } + return nil } diff --git a/commands/release/create/create_test.go b/commands/release/create/create_test.go index 8a38958c2f6f4f866520213416fb4868b7420bd3..f6d3e535ad725f7bff7f7a400fe2d5fd1e784deb 100644 --- a/commands/release/create/create_test.go +++ b/commands/release/create/create_test.go @@ -3,9 +3,12 @@ package create import ( "io" "net/http" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/cli/pkg/httpmock" "gitlab.com/gitlab-org/cli/commands/cmdtest" @@ -307,3 +310,122 @@ func TestReleaseCreate_WithAssetsLinksJSON(t *testing.T) { }) } } + +func TestReleaseCreateWithPublishToCatalog(t *testing.T) { + tests := []struct { + name string + cli string + + wantOutput string + wantBody string + wantErr bool + errMsg string + }{ + { + name: "with version", + cli: "0.0.1 --publish-to-catalog", + wantBody: `{ + "version": "0.0.1", + "metadata": { + "components": [ + { + "component_type": "template", + "name": "component-1", + "spec": { + "inputs": { + "compiler": { + "default": "gcc" + } + } + } + }, + { + "component_type": "template", + "name": "component-2", + "spec": null + }, + { + "component_type": "template", + "name": "component-3", + "spec": { + "inputs": { + "test_framework": { + "default": "unittest" + } + } + } + } + ] + } + }`, + wantOutput: `• Publishing release tag=0.0.1 to the GitLab CI/CD catalog for repo=OWNER/REPO... +✓ Release published: url=https://gitlab.example.com/explore/catalog/my-namespace/my-component-project`, + }, + } + + originalWd, err := os.Getwd() + require.NoError(t, err) + + err = os.Chdir(filepath.Join(originalWd, "catalog", "testdata", "test-repo")) + require.NoError(t, err) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakeHTTP := &httpmock.Mocker{ + MatchURL: httpmock.PathAndQuerystring, + } + defer fakeHTTP.Verify(t) + + fakeHTTP.RegisterResponder(http.MethodGet, "/api/v4/projects/OWNER/REPO/releases/0%2E0%2E1", + httpmock.NewStringResponse(http.StatusNotFound, `{"message":"404 Not Found"}`)) + + fakeHTTP.RegisterResponder(http.MethodPost, "/api/v4/projects/OWNER/REPO/releases", + func(req *http.Request) (*http.Response, error) { + resp, _ := httpmock.NewStringResponse(http.StatusCreated, + `{ + "name": "test_release", + "tag_name": "0.0.1", + "description": "bugfix release", + "created_at": "2023-01-19T02:58:32.622Z", + "released_at": "2023-01-19T02:58:32.622Z", + "upcoming_release": false, + "tag_path": "/OWNER/REPO/-/tags/0.0.1", + "_links": { + "self": "https://gitlab.com/OWNER/REPO/-/releases/0.0.1" + } + }`)(req) + return resp, nil + }, + ) + + if tc.wantBody != "" { + fakeHTTP.RegisterResponder(http.MethodPost, "/api/v4/projects/OWNER/REPO/catalog/publish", + func(req *http.Request) (*http.Response, error) { + body, _ := io.ReadAll(req.Body) + + assert.JSONEq(t, tc.wantBody, string(body)) + + response := httpmock.NewJSONResponse(http.StatusOK, map[string]interface{}{ + "catalog_url": "https://gitlab.example.com/explore/catalog/my-namespace/my-component-project", + }) + + return response(req) + }, + ) + } + + output, err := runCommand(fakeHTTP, false, tc.cli) + + if tc.wantErr { + assert.Error(t, err) + assert.Equal(t, tc.errMsg, err.Error()) + } else { + assert.NoError(t, err) + assert.Contains(t, output.Stderr(), tc.wantOutput) + } + }) + } + + err = os.Chdir(originalWd) + require.NoError(t, err) +} diff --git a/docs/source/release/create.md b/docs/source/release/create.md index 3a53d8d54cb7d2de1d83927bf8b149437ef5ea6c..8339d15950b3080376f915836a84860750dc19ff 100644 --- a/docs/source/release/create.md +++ b/docs/source/release/create.md @@ -68,6 +68,22 @@ $ glab release create v1.0.1 --assets-links=' } ]' +# [EXPERIMENTAL] Create a release and publish it to the GitLab CI/CD catalog +# This command should NOT be run manually, but rather as part of a CI/CD pipeline with the "release" keyword. +# The API endpoint accepts only "CI_JOB_TOKEN" as the authentication token. +# This command retrieves components from the current repository by searching for `yml` files +# within the "templates" directory and its subdirectories. +# This flag will not work if the feature flag `ci_release_cli_catalog_publish_option` is not enabled +# for the project in the GitLab instance. + +# Components can be defined; + +# - In single files ending in `.yml` for each component, like `templates/secret-detection.yml`. +# - In sub-directories containing `template.yml` files as entry points, +# for components that bundle together multiple related files. For example, +# `templates/secret-detection/template.yml`. +$ glab release create v1.0.1 --publish-to-catalog + ``` ## Options @@ -78,6 +94,7 @@ $ glab release create v1.0.1 --assets-links=' -n, --name string The release name or title. -N, --notes string The release notes or description. You can use Markdown. -F, --notes-file string Read release notes 'file'. Specify '-' as the value to read from stdin. + --publish-to-catalog [EXPERIMENTAL] Publish the release to the GitLab CI/CD catalog. -r, --ref string If the specified tag doesn't exist, the release is created from ref and tagged with the specified tag name. It can be a commit SHA, another tag name, or a branch name. -D, --released-at string The 'date' when the release was ready. Defaults to the current datetime. Expects ISO 8601 format (2019-03-15T08:00:00Z). -T, --tag-message string Message to use if creating a new annotated tag.