From efa259aab39dd521c30dec59f7b47e313a9cba76 Mon Sep 17 00:00:00 2001 From: Divya Rani Date: Tue, 23 Jul 2024 12:27:27 +0530 Subject: [PATCH] Add gitaly git subcmd Git binaries are now embeded into the Gitaly binary which allows Gitaly to be in total control of the execution environment. This commit introduces gitaly git subcommand which allows user to execute Git commands via the same Git execution environment as Gitaly. --- cmd/gitaly/main_test.go | 2 +- internal/cli/gitaly/app.go | 1 + internal/cli/gitaly/serve.go | 4 +- internal/cli/gitaly/subcmd_bundleuri.go | 7 +- internal/cli/gitaly/subcmd_check.go | 4 +- internal/cli/gitaly/subcmd_git.go | 109 ++++++++++++++++++++++++ internal/cli/gitaly/subcmd_git_test.go | 107 +++++++++++++++++++++++ internal/cli/gitaly/subcmd_hooks.go | 7 +- packed_binaries.go | 5 +- packed_binaries_test.go | 5 +- 10 files changed, 232 insertions(+), 19 deletions(-) create mode 100644 internal/cli/gitaly/subcmd_git.go create mode 100644 internal/cli/gitaly/subcmd_git_test.go diff --git a/cmd/gitaly/main_test.go b/cmd/gitaly/main_test.go index 4dea4ad0fc..8161dc1838 100644 --- a/cmd/gitaly/main_test.go +++ b/cmd/gitaly/main_test.go @@ -31,7 +31,7 @@ func TestGitalyCLI(t *testing.T) { { desc: "without arguments", exitCode: 2, - stdout: "NAME:\n gitaly - a Git RPC service\n\nUSAGE:\n gitaly command [command options]\n\nDESCRIPTION:\n Gitaly is a Git RPC service for handling Git calls.\n\nCOMMANDS:\n serve launch the server daemon\n check verify internal API is accessible\n configuration run configuration-related commands\n hooks manage Git hooks\n bundle-uri Generate bundle URI bundle\n\nOPTIONS:\n --help, -h show help\n --version, -v print the version\n", + stdout: "NAME:\n gitaly - a Git RPC service\n\nUSAGE:\n gitaly command [command options]\n\nDESCRIPTION:\n Gitaly is a Git RPC service for handling Git calls.\n\nCOMMANDS:\n serve launch the server daemon\n check verify internal API is accessible\n configuration run configuration-related commands\n hooks manage Git hooks\n bundle-uri Generate bundle URI bundle\n git execute Git commands using Gitaly's embedded Git\n\nOPTIONS:\n --help, -h show help\n --version, -v print the version\n", }, { desc: "with non-existent config", diff --git a/internal/cli/gitaly/app.go b/internal/cli/gitaly/app.go index a6976d96ce..02212f94b2 100644 --- a/internal/cli/gitaly/app.go +++ b/internal/cli/gitaly/app.go @@ -33,6 +33,7 @@ func NewApp() *cli.App { newConfigurationCommand(), newHooksCommand(), newBundleURICommand(), + newGitCommand(), }, } } diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index c6b8e1eb11..dcd35c3297 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -221,7 +221,9 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { }() began = time.Now() - if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil { + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(string) bool { + return true + }); err != nil { return fmt.Errorf("unpack auxiliary binaries: %w", err) } logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished unpacking auxiliary binaries") diff --git a/internal/cli/gitaly/subcmd_bundleuri.go b/internal/cli/gitaly/subcmd_bundleuri.go index ab66f80d28..b6c92b5807 100644 --- a/internal/cli/gitaly/subcmd_bundleuri.go +++ b/internal/cli/gitaly/subcmd_bundleuri.go @@ -28,12 +28,7 @@ Example: gitaly bundle-uri --storage=default --repository=ab/cd/ef01234567890123 Usage: "repository to generate bundle-URI for", Required: true, }, - &cli.StringFlag{ - Name: flagConfig, - Usage: "path to Gitaly configuration", - Aliases: []string{"c"}, - Required: true, - }, + gitalyConfigFlag(), }, } } diff --git a/internal/cli/gitaly/subcmd_check.go b/internal/cli/gitaly/subcmd_check.go index 31ab8adc92..ce76de3558 100644 --- a/internal/cli/gitaly/subcmd_check.go +++ b/internal/cli/gitaly/subcmd_check.go @@ -48,7 +48,9 @@ func checkAction(ctx *cli.Context) error { // Since this subcommand invokes a Git command, we need to unpack the bundled Git binaries // from the Gitaly binary. - if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil { + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(string) bool { + return true + }); err != nil { return fmt.Errorf("unpack auxiliary binaries: %w", err) } diff --git a/internal/cli/gitaly/subcmd_git.go b/internal/cli/gitaly/subcmd_git.go new file mode 100644 index 0000000000..f236691a28 --- /dev/null +++ b/internal/cli/gitaly/subcmd_git.go @@ -0,0 +1,109 @@ +package gitaly + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" +) + +func gitalyConfigFlag() *cli.StringFlag { + return &cli.StringFlag{ + Name: flagConfig, + Usage: "path to Gitaly configuration", + Aliases: []string{"c"}, + Required: true, + } +} + +func newGitCommand() *cli.Command { + return &cli.Command{ + Name: "git", + Usage: "execute Git commands using Gitaly's embedded Git", + UsageText: `gitaly git -c -- [git-command] [args...] + +Example: gitaly git -c -- status`, + Description: `=== WARNING === +Do not execute commands in Gitaly's storages +without understanding the implications of doing so. +Modifying Gitaly's state may lead to violating Gitaly's +invariants, and lead to unavailability or data loss. +===============`, + Action: gitAction, + HideHelpCommand: true, + Flags: []cli.Flag{ + gitalyConfigFlag(), + }, + } +} + +func gitAction(ctx *cli.Context) (returnErr error) { + logger := log.ConfigureCommand() + + cfg, err := loadConfig(ctx.String(flagConfig)) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + runtimeDir, err := os.MkdirTemp("", "gitaly-git-*") + if err != nil { + return fmt.Errorf("creating runtime dir: %w", err) + } + + defer func() { + if err := os.RemoveAll(runtimeDir); err != nil { + returnErr = errors.Join(returnErr, fmt.Errorf("removing runtime dir: %w", err)) + } + }() + + cfg.RuntimeDir = runtimeDir + + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(binaryName string) bool { + return strings.HasPrefix(binaryName, "gitaly-git") + }); err != nil { + return fmt.Errorf("unpack auxiliary binaries: %w", err) + } + + gitCmdFactory, cleanup, err := git.NewExecCommandFactory(cfg, logger) + if err != nil { + return fmt.Errorf("creating Git command factory: %w", err) + } + defer cleanup() + + gitBinaryPath := gitCmdFactory.GetExecutionEnvironment(ctx.Context).BinaryPath + + cmd := exec.Command(gitBinaryPath, ctx.Args().Slice()...) + cmd.Stdin = ctx.App.Reader + cmd.Stdout = ctx.App.Writer + cmd.Stderr = ctx.App.ErrWriter + + // Disable automatic garbage collection and maintenance + gitConfig := []git.ConfigPair{ + {Key: "gc.auto", Value: "0"}, + {Key: "maintenance.auto", Value: "0"}, + } + + cmd.Env = os.Environ() + + cmd.Env = append(cmd.Env, + fmt.Sprintf("GIT_EXEC_PATH=%s", filepath.Dir(gitBinaryPath))) + cmd.Env = append(cmd.Env, git.ConfigPairsToGitEnvironment(gitConfig)...) + + err = cmd.Run() + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return cli.Exit("", exitError.ExitCode()) + } + return fmt.Errorf("executing git command: %w", err) + } + + return nil +} diff --git a/internal/cli/gitaly/subcmd_git_test.go b/internal/cli/gitaly/subcmd_git_test.go new file mode 100644 index 0000000000..f9edbcc4ef --- /dev/null +++ b/internal/cli/gitaly/subcmd_git_test.go @@ -0,0 +1,107 @@ +package gitaly + +import ( + "bytes" + "os" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" +) + +func TestGitalyGitCommand(t *testing.T) { + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + cfg.Git.BinPath = os.Getenv("GITALY_TESTING_GIT_BINARY") + + testcfg.BuildGitaly(t, cfg) + _, repo := gittest.CreateRepository(t, ctx, cfg, + gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + + commitID := gittest.WriteCommit(t, cfg, repo, + gittest.WithBranch("main"), + gittest.WithMessage("First commit"), + gittest.WithTreeEntries( + gittest.TreeEntry{Mode: "100644", Path: "README.md", Content: "# Initial content"}, + ), + ) + + configPath := testcfg.WriteTemporaryGitalyConfigFile(t, cfg) + + tests := []struct { + name string + args []string + input string + pipeCommand *exec.Cmd + exitCode int + expectedOutput string + expectedError string + }{ + { + name: "git log", + args: []string{"log", "-1", "--pretty=format:%s"}, + expectedOutput: "First commit", + }, + { + name: "git rev-list with --stdin", + args: []string{"rev-list", "--stdin"}, + input: "HEAD", + expectedOutput: commitID.String() + "\n", + }, + { + name: "git rev-parse --is-inside-work-tree", + args: []string{"rev-parse", "--is-inside-work-tree"}, + expectedOutput: "false\n", + }, + { + name: "invalid git command", + args: []string{"invalid-command"}, + exitCode: 1, + expectedError: "git: 'invalid-command' is not a git command. See 'git --help'.\n", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + args := append([]string{"git", "--config", configPath}, tt.args...) + cmd := exec.Command(cfg.BinaryPath("gitaly"), args...) + + var stdout, stderr bytes.Buffer + cmd.Dir = repo + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + if tt.input != "" { + cmd.Stdin = strings.NewReader(tt.input) + } + + // Override these environment variables to avoid interference with the test. + cmd.Env = []string{ + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "XDG_CONFIG_HOME=/dev/null", + } + + err := cmd.Run() + + if tt.expectedError != "" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, tt.expectedError, stderr.String()) + require.Equal(t, tt.exitCode, cmd.ProcessState.ExitCode()) + require.Equal(t, tt.expectedOutput, stdout.String()) + }) + } +} diff --git a/internal/cli/gitaly/subcmd_hooks.go b/internal/cli/gitaly/subcmd_hooks.go index 184a153a5e..9b81003f35 100644 --- a/internal/cli/gitaly/subcmd_hooks.go +++ b/internal/cli/gitaly/subcmd_hooks.go @@ -52,12 +52,7 @@ To remove custom Git hooks for a specified repository, run the set subcommand wi Usage: "repository to set hooks for", Required: true, }, - &cli.StringFlag{ - Name: flagConfig, - Usage: "path to Gitaly configuration", - Aliases: []string{"c"}, - Required: true, - }, + gitalyConfigFlag(), }, }, }, diff --git a/packed_binaries.go b/packed_binaries.go index 772eb5309c..540d5e3c77 100644 --- a/packed_binaries.go +++ b/packed_binaries.go @@ -32,7 +32,7 @@ var packedBinariesFS embed.FS // binaries in the main gitaly binary and unpacking them on start to a temporary directory we can call them from. This // way updating the gitaly binaries on the disk is atomic and a running gitaly can't call auxiliary binaries from a // different version. -func UnpackAuxiliaryBinaries(destinationDir string) error { +func UnpackAuxiliaryBinaries(destinationDir string, shouldInclude func(binaryName string) bool) error { entries, err := packedBinariesFS.ReadDir(buildDir) if err != nil { return fmt.Errorf("list packed binaries: %w", err) @@ -41,6 +41,9 @@ func UnpackAuxiliaryBinaries(destinationDir string) error { g := &errgroup.Group{} for _, entry := range entries { entry := entry + if !shouldInclude(entry.Name()) { + continue + } g.Go(func() error { packedPath := filepath.Join(buildDir, entry.Name()) packedFile, err := packedBinariesFS.Open(packedPath) diff --git a/packed_binaries_test.go b/packed_binaries_test.go index 952fe19706..ff845a0e4f 100644 --- a/packed_binaries_test.go +++ b/packed_binaries_test.go @@ -12,7 +12,7 @@ import ( func TestUnpackAuxiliaryBinaries_success(t *testing.T) { destinationDir := t.TempDir() - require.NoError(t, UnpackAuxiliaryBinaries(destinationDir)) + require.NoError(t, UnpackAuxiliaryBinaries(destinationDir, func(string) bool { return true })) entries, err := os.ReadDir(destinationDir) require.NoError(t, err) @@ -36,10 +36,9 @@ func TestUnpackAuxiliaryBinaries_success(t *testing.T) { func TestUnpackAuxiliaryBinaries_alreadyExists(t *testing.T) { destinationDir := t.TempDir() - existingFile := filepath.Join(destinationDir, "gitaly-hooks") require.NoError(t, os.WriteFile(existingFile, []byte("existing file"), mode.File)) - err := UnpackAuxiliaryBinaries(destinationDir) + err := UnpackAuxiliaryBinaries(destinationDir, func(_ string) bool { return true }) require.EqualError(t, err, fmt.Sprintf(`open %s: file exists`, existingFile), "expected unpacking to fail if destination binary already existed") } -- GitLab