diff --git a/go.mod b/go.mod index d4845d5ce4ae530204e0af9167ac0c6f2db38f9b..a3d25ec3f71d47aa0f2683f515b106acd19f07fb 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,8 @@ require ( google.golang.org/protobuf v1.36.3 ) +require github.com/pelletier/go-toml v1.9.5 + require ( cel.dev/expr v0.16.1 // indirect cloud.google.com/go v0.115.1 // indirect diff --git a/go.sum b/go.sum index 6ea0ad2fc2e95e769f3d9388b295171be5b19408..49044b3848b945434628fb2732328212be4dd4f2 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= diff --git a/internal/cli/gitalybackup/create.go b/internal/cli/gitalybackup/create.go index 4127f4e6454a79e8cd6f7327f404f208f0f754e7..03db764e20e7e4328c697c5922a49a6bf02f1b89 100644 --- a/internal/cli/gitalybackup/create.go +++ b/internal/cli/gitalybackup/create.go @@ -6,9 +6,12 @@ import ( "errors" "fmt" "io" + "os" + "path/filepath" "runtime" "time" + "github.com/pelletier/go-toml" cli "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" @@ -17,6 +20,12 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) +const gitlabConfigPath = "/var/opt/gitlab/gitaly/config.toml" + +type storageInfo struct { + StorageName string + RelativePath string +} type serverRepository struct { storage.ServerInfo StorageName string `json:"storage_name"` @@ -32,6 +41,7 @@ type createSubcommand struct { incremental bool backupID string serverSide bool + backupupAll bool } func (cmd *createSubcommand) flags(ctx *cli.Context) { @@ -42,6 +52,7 @@ func (cmd *createSubcommand) flags(ctx *cli.Context) { cmd.incremental = ctx.Bool("incremental") cmd.backupID = ctx.String("id") cmd.serverSide = ctx.Bool("server-side") + cmd.backupupAll = ctx.Bool("all-repositories") } func createFlags() []cli.Flag { @@ -152,7 +163,46 @@ func (cmd *createSubcommand) run(ctx context.Context, logger log.Logger, stdin i return fmt.Errorf("create pipeline: %w", err) } + if cmd.backupupAll { + return cmd.pipeAll(ctx, manager, pipeline) + } decoder := json.NewDecoder(stdin) + return cmd.pipeEntry(ctx, decoder, manager, pipeline) + +} + +func (cmd *createSubcommand) pipeAll(ctx context.Context, manager backup.Strategy, pipeline *backup.Pipeline) error { + + serverRepositories, err := getAllRepositories() + if err != nil { + return fmt.Errorf("create: get all repositories: %w", err) + } + for _, sr := range serverRepositories { + + repo := gitalypb.Repository{ + StorageName: sr.StorageName, + RelativePath: sr.RelativePath, + GlProjectPath: sr.GlProjectPath, + } + pipeline.Handle(ctx, backup.NewCreateCommand(manager, backup.CreateRequest{ + Server: sr.ServerInfo, + Repository: &repo, + VanityRepository: &repo, + Incremental: cmd.incremental, + BackupID: cmd.backupID, + }, + )) + } + + if _, err := pipeline.Done(); err != nil { + return fmt.Errorf("create: %w", err) + + } + return nil +} + +func (cmd *createSubcommand) pipeEntry(ctx context.Context, decoder *json.Decoder, manager backup.Strategy, pipeline *backup.Pipeline) error { + for { var sr serverRepository if err := decoder.Decode(&sr); errors.Is(err, io.EOF) { @@ -179,3 +229,72 @@ func (cmd *createSubcommand) run(ctx context.Context, logger log.Logger, stdin i } return nil } + +func getHashedStoragePaths() ([]storageInfo, error) { + var repoInfo []storageInfo + + configData, err := os.ReadFile(gitlabConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + config, err := toml.Load(string(configData)) + if err != nil { + return nil, fmt.Errorf("failed to parse TOML: %w", err) + } + + storagesRaw := config.Get("storage") + if storagesRaw == nil { + return nil, fmt.Errorf("storage section not found in config") + } + + storages, ok := storagesRaw.([]*toml.Tree) + if !ok { + return nil, fmt.Errorf("failed to parse storage section") + } + + for _, storage := range storages { + name, nameOK := storage.Get("name").(string) + path, pathOK := storage.Get("path").(string) + + if nameOK && pathOK { + repoInfo = append(repoInfo, storageInfo{ + StorageName: name, + RelativePath: filepath.Join(path, "@hashed"), // Ensure we scan @hashed + }) + } + } + + return repoInfo, nil +} + +func getAllRepositories() ([]serverRepository, error) { + repoHashesInfo, err := getHashedStoragePaths() + if err != nil { + return nil, err + } + + var serverRepositories []serverRepository + + for _, repoHashInfo := range repoHashesInfo { + + err := filepath.WalkDir(repoHashInfo.RelativePath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + if filepath.Ext(path) == ".git" && !d.IsDir() { + serverRepositories = append(serverRepositories, serverRepository{StorageName: repoHashInfo.StorageName, + RelativePath: path, + GlProjectPath: ""}) + } + return nil + }) + + if err != nil { + return nil, err + } + } + + return serverRepositories, nil +}