diff --git a/cmd/gitaly/main_test.go b/cmd/gitaly/main_test.go index 4dea4ad0fcb86b3f96925aecef1e32d99bcf707e..8161dc18385bac19f73390725898c8a523c62f0e 100644 --- a/cmd/gitaly/main_test.go +++ b/cmd/gitaly/main_test.go @@ -31,7 +31,7 @@ func TestGitalyCLI(t *testing.T) { { desc: "without arguments", exitCode: 2, - stdout: "NAME:\n gitaly - a Git RPC service\n\nUSAGE:\n gitaly command [command options]\n\nDESCRIPTION:\n Gitaly is a Git RPC service for handling Git calls.\n\nCOMMANDS:\n serve launch the server daemon\n check verify internal API is accessible\n configuration run configuration-related commands\n hooks manage Git hooks\n bundle-uri Generate bundle URI bundle\n\nOPTIONS:\n --help, -h show help\n --version, -v print the version\n", + stdout: "NAME:\n gitaly - a Git RPC service\n\nUSAGE:\n gitaly command [command options]\n\nDESCRIPTION:\n Gitaly is a Git RPC service for handling Git calls.\n\nCOMMANDS:\n serve launch the server daemon\n check verify internal API is accessible\n configuration run configuration-related commands\n hooks manage Git hooks\n bundle-uri Generate bundle URI bundle\n git execute Git commands using Gitaly's embedded Git\n\nOPTIONS:\n --help, -h show help\n --version, -v print the version\n", }, { desc: "with non-existent config", diff --git a/internal/cli/gitaly/app.go b/internal/cli/gitaly/app.go index a6976d96ce536dcc22defabc255ef83ac5ab0b5d..02212f94b2b6fa8ffef5aca54f246206449fbf4d 100644 --- a/internal/cli/gitaly/app.go +++ b/internal/cli/gitaly/app.go @@ -33,6 +33,7 @@ func NewApp() *cli.App { newConfigurationCommand(), newHooksCommand(), newBundleURICommand(), + newGitCommand(), }, } } diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index c6b8e1eb11f8acd6afff29f7c93b78c9320cc7dc..dcd35c32977395ee3344f870b32ca0c29ed2d8be 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -221,7 +221,9 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { }() began = time.Now() - if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil { + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(string) bool { + return true + }); err != nil { return fmt.Errorf("unpack auxiliary binaries: %w", err) } logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished unpacking auxiliary binaries") diff --git a/internal/cli/gitaly/subcmd_bundleuri.go b/internal/cli/gitaly/subcmd_bundleuri.go index ab66f80d28e0a94ba9f152c7f8bde8fa1dd3f7e9..b6c92b5807537119b237d6df7096b5221323ce23 100644 --- a/internal/cli/gitaly/subcmd_bundleuri.go +++ b/internal/cli/gitaly/subcmd_bundleuri.go @@ -28,12 +28,7 @@ Example: gitaly bundle-uri --storage=default --repository=ab/cd/ef01234567890123 Usage: "repository to generate bundle-URI for", Required: true, }, - &cli.StringFlag{ - Name: flagConfig, - Usage: "path to Gitaly configuration", - Aliases: []string{"c"}, - Required: true, - }, + gitalyConfigFlag(), }, } } diff --git a/internal/cli/gitaly/subcmd_check.go b/internal/cli/gitaly/subcmd_check.go index 31ab8adc92fecd9f057c77b8c440a7c46b049367..ce76de35581f8eaf2032c993a68050b99084d890 100644 --- a/internal/cli/gitaly/subcmd_check.go +++ b/internal/cli/gitaly/subcmd_check.go @@ -48,7 +48,9 @@ func checkAction(ctx *cli.Context) error { // Since this subcommand invokes a Git command, we need to unpack the bundled Git binaries // from the Gitaly binary. - if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil { + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(string) bool { + return true + }); err != nil { return fmt.Errorf("unpack auxiliary binaries: %w", err) } diff --git a/internal/cli/gitaly/subcmd_git.go b/internal/cli/gitaly/subcmd_git.go new file mode 100644 index 0000000000000000000000000000000000000000..f236691a281232076ae652d451ea22f38fc832e4 --- /dev/null +++ b/internal/cli/gitaly/subcmd_git.go @@ -0,0 +1,109 @@ +package gitaly + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" +) + +func gitalyConfigFlag() *cli.StringFlag { + return &cli.StringFlag{ + Name: flagConfig, + Usage: "path to Gitaly configuration", + Aliases: []string{"c"}, + Required: true, + } +} + +func newGitCommand() *cli.Command { + return &cli.Command{ + Name: "git", + Usage: "execute Git commands using Gitaly's embedded Git", + UsageText: `gitaly git -c -- [git-command] [args...] + +Example: gitaly git -c -- status`, + Description: `=== WARNING === +Do not execute commands in Gitaly's storages +without understanding the implications of doing so. +Modifying Gitaly's state may lead to violating Gitaly's +invariants, and lead to unavailability or data loss. +===============`, + Action: gitAction, + HideHelpCommand: true, + Flags: []cli.Flag{ + gitalyConfigFlag(), + }, + } +} + +func gitAction(ctx *cli.Context) (returnErr error) { + logger := log.ConfigureCommand() + + cfg, err := loadConfig(ctx.String(flagConfig)) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + runtimeDir, err := os.MkdirTemp("", "gitaly-git-*") + if err != nil { + return fmt.Errorf("creating runtime dir: %w", err) + } + + defer func() { + if err := os.RemoveAll(runtimeDir); err != nil { + returnErr = errors.Join(returnErr, fmt.Errorf("removing runtime dir: %w", err)) + } + }() + + cfg.RuntimeDir = runtimeDir + + if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(binaryName string) bool { + return strings.HasPrefix(binaryName, "gitaly-git") + }); err != nil { + return fmt.Errorf("unpack auxiliary binaries: %w", err) + } + + gitCmdFactory, cleanup, err := git.NewExecCommandFactory(cfg, logger) + if err != nil { + return fmt.Errorf("creating Git command factory: %w", err) + } + defer cleanup() + + gitBinaryPath := gitCmdFactory.GetExecutionEnvironment(ctx.Context).BinaryPath + + cmd := exec.Command(gitBinaryPath, ctx.Args().Slice()...) + cmd.Stdin = ctx.App.Reader + cmd.Stdout = ctx.App.Writer + cmd.Stderr = ctx.App.ErrWriter + + // Disable automatic garbage collection and maintenance + gitConfig := []git.ConfigPair{ + {Key: "gc.auto", Value: "0"}, + {Key: "maintenance.auto", Value: "0"}, + } + + cmd.Env = os.Environ() + + cmd.Env = append(cmd.Env, + fmt.Sprintf("GIT_EXEC_PATH=%s", filepath.Dir(gitBinaryPath))) + cmd.Env = append(cmd.Env, git.ConfigPairsToGitEnvironment(gitConfig)...) + + err = cmd.Run() + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return cli.Exit("", exitError.ExitCode()) + } + return fmt.Errorf("executing git command: %w", err) + } + + return nil +} diff --git a/internal/cli/gitaly/subcmd_git_test.go b/internal/cli/gitaly/subcmd_git_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f9edbcc4efe050987d3fa8198e6bcfae60879e84 --- /dev/null +++ b/internal/cli/gitaly/subcmd_git_test.go @@ -0,0 +1,107 @@ +package gitaly + +import ( + "bytes" + "os" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" +) + +func TestGitalyGitCommand(t *testing.T) { + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + cfg.Git.BinPath = os.Getenv("GITALY_TESTING_GIT_BINARY") + + testcfg.BuildGitaly(t, cfg) + _, repo := gittest.CreateRepository(t, ctx, cfg, + gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + + commitID := gittest.WriteCommit(t, cfg, repo, + gittest.WithBranch("main"), + gittest.WithMessage("First commit"), + gittest.WithTreeEntries( + gittest.TreeEntry{Mode: "100644", Path: "README.md", Content: "# Initial content"}, + ), + ) + + configPath := testcfg.WriteTemporaryGitalyConfigFile(t, cfg) + + tests := []struct { + name string + args []string + input string + pipeCommand *exec.Cmd + exitCode int + expectedOutput string + expectedError string + }{ + { + name: "git log", + args: []string{"log", "-1", "--pretty=format:%s"}, + expectedOutput: "First commit", + }, + { + name: "git rev-list with --stdin", + args: []string{"rev-list", "--stdin"}, + input: "HEAD", + expectedOutput: commitID.String() + "\n", + }, + { + name: "git rev-parse --is-inside-work-tree", + args: []string{"rev-parse", "--is-inside-work-tree"}, + expectedOutput: "false\n", + }, + { + name: "invalid git command", + args: []string{"invalid-command"}, + exitCode: 1, + expectedError: "git: 'invalid-command' is not a git command. See 'git --help'.\n", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + args := append([]string{"git", "--config", configPath}, tt.args...) + cmd := exec.Command(cfg.BinaryPath("gitaly"), args...) + + var stdout, stderr bytes.Buffer + cmd.Dir = repo + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + if tt.input != "" { + cmd.Stdin = strings.NewReader(tt.input) + } + + // Override these environment variables to avoid interference with the test. + cmd.Env = []string{ + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "XDG_CONFIG_HOME=/dev/null", + } + + err := cmd.Run() + + if tt.expectedError != "" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, tt.expectedError, stderr.String()) + require.Equal(t, tt.exitCode, cmd.ProcessState.ExitCode()) + require.Equal(t, tt.expectedOutput, stdout.String()) + }) + } +} diff --git a/internal/cli/gitaly/subcmd_hooks.go b/internal/cli/gitaly/subcmd_hooks.go index 184a153a5e3377e3a46e91f69b7506a2db7f9bef..9b81003f3512e2a38fd09f3461979cdc57bd0c3c 100644 --- a/internal/cli/gitaly/subcmd_hooks.go +++ b/internal/cli/gitaly/subcmd_hooks.go @@ -52,12 +52,7 @@ To remove custom Git hooks for a specified repository, run the set subcommand wi Usage: "repository to set hooks for", Required: true, }, - &cli.StringFlag{ - Name: flagConfig, - Usage: "path to Gitaly configuration", - Aliases: []string{"c"}, - Required: true, - }, + gitalyConfigFlag(), }, }, }, diff --git a/packed_binaries.go b/packed_binaries.go index 772eb5309c4cd2d7843d8eb0edd32c93f5dc3163..540d5e3c773501834d49234211c54044c8d10967 100644 --- a/packed_binaries.go +++ b/packed_binaries.go @@ -32,7 +32,7 @@ var packedBinariesFS embed.FS // binaries in the main gitaly binary and unpacking them on start to a temporary directory we can call them from. This // way updating the gitaly binaries on the disk is atomic and a running gitaly can't call auxiliary binaries from a // different version. -func UnpackAuxiliaryBinaries(destinationDir string) error { +func UnpackAuxiliaryBinaries(destinationDir string, shouldInclude func(binaryName string) bool) error { entries, err := packedBinariesFS.ReadDir(buildDir) if err != nil { return fmt.Errorf("list packed binaries: %w", err) @@ -41,6 +41,9 @@ func UnpackAuxiliaryBinaries(destinationDir string) error { g := &errgroup.Group{} for _, entry := range entries { entry := entry + if !shouldInclude(entry.Name()) { + continue + } g.Go(func() error { packedPath := filepath.Join(buildDir, entry.Name()) packedFile, err := packedBinariesFS.Open(packedPath) diff --git a/packed_binaries_test.go b/packed_binaries_test.go index 952fe19706947443bbd33a8fcf25f60859229649..ff845a0e4fc262cd842347294f8e570855e40b25 100644 --- a/packed_binaries_test.go +++ b/packed_binaries_test.go @@ -12,7 +12,7 @@ import ( func TestUnpackAuxiliaryBinaries_success(t *testing.T) { destinationDir := t.TempDir() - require.NoError(t, UnpackAuxiliaryBinaries(destinationDir)) + require.NoError(t, UnpackAuxiliaryBinaries(destinationDir, func(string) bool { return true })) entries, err := os.ReadDir(destinationDir) require.NoError(t, err) @@ -36,10 +36,9 @@ func TestUnpackAuxiliaryBinaries_success(t *testing.T) { func TestUnpackAuxiliaryBinaries_alreadyExists(t *testing.T) { destinationDir := t.TempDir() - existingFile := filepath.Join(destinationDir, "gitaly-hooks") require.NoError(t, os.WriteFile(existingFile, []byte("existing file"), mode.File)) - err := UnpackAuxiliaryBinaries(destinationDir) + err := UnpackAuxiliaryBinaries(destinationDir, func(_ string) bool { return true }) require.EqualError(t, err, fmt.Sprintf(`open %s: file exists`, existingFile), "expected unpacking to fail if destination binary already existed") }