diff --git a/docs/source/api/_index.md b/docs/source/api/_index.md index 2423884100c971d70c68913a4db2cd21c918548b..0b703396b6de90c7e25d795fe439a41748dd2d07 100644 --- a/docs/source/api/_index.md +++ b/docs/source/api/_index.md @@ -66,6 +66,12 @@ no more pages of results remain. For GraphQL requests: - The original query must accept an `$endCursor: String` variable. - The query must fetch the `pageInfo{ hasNextPage, endCursor }` set of fields from a collection. +The `--output` flag controls the output format: + +- `json` (default): Pretty-printed JSON. Arrays are output as a single JSON array. +- `ndjson`: Newline-delimited JSON. Each array element or object is output on a separate line. + This format is more memory-efficient for large datasets and works well with tools like `jq`. + ```plaintext glab api [flags] ``` @@ -76,6 +82,8 @@ glab api [flags] $ glab api projects/:fullpath/releases $ glab api projects/gitlab-com%2Fwww-gitlab-com/issues $ glab api issues --paginate +$ glab api issues --paginate --output ndjson +$ glab api issues --paginate --output ndjson | jq 'select(.state == "opened")' $ glab api graphql -f query="query { currentUser { username } }" $ glab api graphql -f query=' query { @@ -125,6 +133,7 @@ $ glab api graphql --paginate -f query=' -i, --include Include HTTP response headers in the output. --input string The file to use as the body for the HTTP request. -X, --method string The HTTP method for the request. (default "GET") + --output string Format output as: json, ndjson. (default "json") --paginate Make additional HTTP requests to fetch all pages of results. -f, --raw-field stringArray Add a string parameter. --silent Do not print the response body. diff --git a/internal/commands/api/api.go b/internal/commands/api/api.go index 16d8d0a4f2d9fcccf3db91f5e19ac9b922a37032..4e81706f689676458fbf8ff153559de716a2bfcf 100644 --- a/internal/commands/api/api.go +++ b/internal/commands/api/api.go @@ -45,6 +45,7 @@ type options struct { showResponseHeaders bool paginate bool silent bool + outputFormat string } func NewCmdApi(f cmdutils.Factory, runF func(*options) error) *cobra.Command { @@ -110,11 +111,19 @@ func NewCmdApi(f cmdutils.Factory, runF func(*options) error) *cobra.Command { - The original query must accept an %[1]s$endCursor: String%[1]s variable. - The query must fetch the %[1]spageInfo{ hasNextPage, endCursor }%[1]s set of fields from a collection. + + The %[1]s--output%[1]s flag controls the output format: + + - %[1]sjson%[1]s (default): Pretty-printed JSON. Arrays are output as a single JSON array. + - %[1]sndjson%[1]s: Newline-delimited JSON. Each array element or object is output on a separate line. + This format is more memory-efficient for large datasets and works well with tools like %[1]sjq%[1]s. `, "`"), Example: heredoc.Doc(` $ glab api projects/:fullpath/releases $ glab api projects/gitlab-com%2Fwww-gitlab-com/issues $ glab api issues --paginate + $ glab api issues --paginate --output ndjson + $ glab api issues --paginate --output ndjson | jq 'select(.state == "opened")' $ glab api graphql -f query="query { currentUser { username } }" $ glab api graphql -f query=' query { @@ -184,6 +193,7 @@ func NewCmdApi(f cmdutils.Factory, runF func(*options) error) *cobra.Command { cmd.Flags().BoolVar(&opts.paginate, "paginate", false, "Make additional HTTP requests to fetch all pages of results.") cmd.Flags().StringVar(&opts.requestInputFile, "input", "", "The file to use as the body for the HTTP request.") cmd.Flags().BoolVar(&opts.silent, "silent", false, "Do not print the response body.") + cmd.Flags().StringVar(&opts.outputFormat, "output", "json", "Format output as: json, ndjson.") cmd.MarkFlagsMutuallyExclusive("paginate", "input") return cmd } @@ -204,6 +214,10 @@ func (o *options) validate(cmd *cobra.Command) error { return &cmdutils.FlagError{Err: errors.New(`the '--paginate' option is not supported for non-GET requests.`)} } + if o.outputFormat != "json" && o.outputFormat != "ndjson" { + return &cmdutils.FlagError{Err: fmt.Errorf("invalid output format %q: must be 'json' or 'ndjson'", o.outputFormat)} + } + return nil } @@ -335,7 +349,10 @@ func processResponse(resp *http.Response, opts *options, headersOutputStream io. } var err error - if isJSON && opts.io.ColorEnabled() { + // Handle NDJSON output format + if opts.outputFormat == "ndjson" && isJSON && resp.StatusCode == http.StatusOK { + err = streamNDJSON(responseBody, opts.io.StdOut) + } else if isJSON && opts.io.ColorEnabled() { out := &bytes.Buffer{} _, err = io.Copy(out, responseBody) if err == nil { @@ -364,6 +381,64 @@ func processResponse(resp *http.Response, opts *options, headersOutputStream io. return "", nil } +// streamNDJSON streams JSON response as newline-delimited JSON. +// If the response is a JSON array, each element is written as a separate line. +// If the response is a single JSON object, it's written as-is with a newline. +func streamNDJSON(body io.Reader, out io.Writer) error { + dec := json.NewDecoder(body) + enc := json.NewEncoder(out) + enc.SetEscapeHTML(false) + + // Peek at the first token to determine if it's an array or object + token, err := dec.Token() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + // Check if it's an array + if delim, ok := token.(json.Delim); ok && delim == '[' { + // Stream each array element as a separate line + for dec.More() { + var element json.RawMessage + if err := dec.Decode(&element); err != nil { + return err + } + if err := enc.Encode(&element); err != nil { + return err + } + } + // Consume the closing bracket + if _, err := dec.Token(); err != nil { + return err + } + return nil + } + + // For non-array JSON, simply copy the entire body and add newline + // Reset the body reader since we consumed the first token + var tokenStr string + switch v := token.(type) { + case json.Delim: + tokenStr = string(rune(v)) + default: + // For primitive values (string, number, bool, null), marshal back to JSON + b, _ := json.Marshal(v) + tokenStr = string(b) + } + + fullBody := io.MultiReader(strings.NewReader(tokenStr), dec.Buffered(), body) + + // Copy directly to output with newline + if _, err := io.Copy(out, fullBody); err != nil { + return err + } + _, err = out.Write([]byte("\n")) + return err +} + var placeholderRE = regexp.MustCompile(`:(group/:namespace/:repo|namespace/:repo|fullpath|id|user|username|group|namespace|repo|branch)\b`) // fillPlaceholders populates `:namespace` and `:repo` placeholders with values from the current repository diff --git a/internal/commands/api/api_test.go b/internal/commands/api/api_test.go index a79532a236d742bbb5202925e87b3392cc9e3984..04bba64260a14b912a71d112ac06e926d8201198 100644 --- a/internal/commands/api/api_test.go +++ b/internal/commands/api/api_test.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "strings" "testing" "github.com/google/shlex" @@ -517,6 +518,7 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { requestMethod: http.MethodPost, requestPath: "graphql", paginate: true, + outputFormat: "json", } err := options.run(t.Context()) @@ -546,6 +548,151 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { assert.Equal(t, "PAGE1_END", endCursor) } +func Test_apiRun_ndjson(t *testing.T) { + t.Parallel() + + ios, _, stdout, stderr := cmdtest.TestIOStreams() + + var tr roundTripFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{`application/json`}}, + Body: io.NopCloser(bytes.NewBufferString(`[{"id":1,"title":"Issue 1"},{"id":2,"title":"Issue 2"}]`)), + Request: req, + }, nil + } + a := cmdtest.NewTestApiClient(t, &http.Client{Transport: tr}, "OTOKEN", "gitlab.com") + options := options{ + io: ios, + baseRepo: func() (glrepo.Interface, error) { + return nil, fmt.Errorf("not supposed to be called") + }, + apiClient: func(repoHost string) (*api.Client, error) { + return a, nil + }, + requestPath: "issues", + outputFormat: "ndjson", + } + + err := options.run(t.Context()) + require.NoError(t, err) + + // NDJSON should output each element on a separate line + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + assert.Equal(t, 2, len(lines), "should have 2 lines") + + // Verify each line is valid JSON + var obj1, obj2 map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &obj1)) + require.NoError(t, json.Unmarshal([]byte(lines[1]), &obj2)) + + assert.Equal(t, float64(1), obj1["id"]) + assert.Equal(t, "Issue 1", obj1["title"]) + assert.Equal(t, float64(2), obj2["id"]) + assert.Equal(t, "Issue 2", obj2["title"]) + assert.Equal(t, "", stderr.String(), "stderr") +} + +func Test_apiRun_ndjson_singleObject(t *testing.T) { + t.Parallel() + + ios, _, stdout, stderr := cmdtest.TestIOStreams() + + var tr roundTripFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{`application/json`}}, + Body: io.NopCloser(bytes.NewBufferString(`{"id":1,"title":"Single Issue"}`)), + Request: req, + }, nil + } + a := cmdtest.NewTestApiClient(t, &http.Client{Transport: tr}, "OTOKEN", "gitlab.com") + options := options{ + io: ios, + baseRepo: func() (glrepo.Interface, error) { + return nil, fmt.Errorf("not supposed to be called") + }, + apiClient: func(repoHost string) (*api.Client, error) { + return a, nil + }, + requestPath: "issues/1", + outputFormat: "ndjson", + } + + err := options.run(t.Context()) + require.NoError(t, err) + + // Single object should be output as one line + output := strings.TrimSpace(stdout.String()) + assert.Equal(t, 1, len(strings.Split(output, "\n")), "should have 1 line") + + var obj map[string]any + require.NoError(t, json.Unmarshal([]byte(output), &obj)) + assert.Equal(t, float64(1), obj["id"]) + assert.Equal(t, "Single Issue", obj["title"]) + assert.Equal(t, "", stderr.String(), "stderr") +} + +func Test_apiRun_ndjson_pagination(t *testing.T) { + t.Parallel() + + ios, _, stdout, stderr := cmdtest.TestIOStreams() + + requestCount := 0 + responses := []*http.Response{ + { + StatusCode: http.StatusOK, + Header: http.Header{ + "Content-Type": []string{`application/json`}, + "Link": []string{`; rel="next"`}, + }, + Body: io.NopCloser(bytes.NewBufferString(`[{"id":1,"title":"Issue 1"}]`)), + }, + { + StatusCode: http.StatusOK, + Header: http.Header{ + "Content-Type": []string{`application/json`}, + }, + Body: io.NopCloser(bytes.NewBufferString(`[{"id":2,"title":"Issue 2"}]`)), + }, + } + + var tr roundTripFunc = func(req *http.Request) (*http.Response, error) { + resp := responses[requestCount] + resp.Request = req + requestCount++ + return resp, nil + } + a := cmdtest.NewTestApiClient(t, &http.Client{Transport: tr}, "OTOKEN", "gitlab.com") + options := options{ + io: ios, + baseRepo: func() (glrepo.Interface, error) { + return nil, fmt.Errorf("not supposed to be called") + }, + apiClient: func(repoHost string) (*api.Client, error) { + return a, nil + }, + requestPath: "issues", + paginate: true, + outputFormat: "ndjson", + } + + err := options.run(t.Context()) + require.NoError(t, err) + + // Should have 2 lines total (one from each page) + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + assert.Equal(t, 2, len(lines), "should have 2 lines from 2 pages") + + var obj1, obj2 map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &obj1)) + require.NoError(t, json.Unmarshal([]byte(lines[1]), &obj2)) + + assert.Equal(t, float64(1), obj1["id"]) + assert.Equal(t, float64(2), obj2["id"]) + assert.Equal(t, "", stderr.String(), "stderr") +} + func Test_apiRun_inputFile(t *testing.T) { tests := []struct { name string