From 9949aac360540879bbdfe895ee4541ade3f9dab4 Mon Sep 17 00:00:00 2001 From: romongbale Date: Mon, 10 Feb 2025 16:26:56 -0500 Subject: [PATCH 1/2] feat: gitaly backup retrieve all repo storage in a node --- go.mod | 2 + go.sum | 2 + internal/cli/gitalybackup/create.go | 119 ++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+) diff --git a/go.mod b/go.mod index d4845d5ce4..a3d25ec3f7 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,8 @@ require ( google.golang.org/protobuf v1.36.3 ) +require github.com/pelletier/go-toml v1.9.5 + require ( cel.dev/expr v0.16.1 // indirect cloud.google.com/go v0.115.1 // indirect diff --git a/go.sum b/go.sum index 6ea0ad2fc2..49044b3848 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= diff --git a/internal/cli/gitalybackup/create.go b/internal/cli/gitalybackup/create.go index 4127f4e645..eacfd0a6fb 100644 --- a/internal/cli/gitalybackup/create.go +++ b/internal/cli/gitalybackup/create.go @@ -6,9 +6,12 @@ import ( "errors" "fmt" "io" + "os" + "path/filepath" "runtime" "time" + "github.com/pelletier/go-toml" cli "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" @@ -17,6 +20,12 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) +const gitlabConfigPath = "/var/opt/gitlab/gitaly/config.toml" + +type storageInfo struct { + StorageName string + RelativePath string +} type serverRepository struct { storage.ServerInfo StorageName string `json:"storage_name"` @@ -32,6 +41,7 @@ type createSubcommand struct { incremental bool backupID string serverSide bool + backupupAll bool } func (cmd *createSubcommand) flags(ctx *cli.Context) { @@ -42,6 +52,7 @@ func (cmd *createSubcommand) flags(ctx *cli.Context) { cmd.incremental = ctx.Bool("incremental") cmd.backupID = ctx.String("id") cmd.serverSide = ctx.Bool("server-side") + cmd.backupupAll = ctx.Bool("all-repositories") } func createFlags() []cli.Flag { @@ -153,6 +164,45 @@ func (cmd *createSubcommand) run(ctx context.Context, logger log.Logger, stdin i } decoder := json.NewDecoder(stdin) + if cmd.backupupAll { + return cmd.pipeAll(ctx, manager, pipeline) + } + return cmd.pipeEntry(ctx, decoder, manager, pipeline) + +} + +func (cmd *createSubcommand) pipeAll(ctx context.Context, manager backup.Strategy, pipeline *backup.Pipeline) error { + + serverRepositories, err := getAllRepositories() + if err != nil { + return fmt.Errorf("create: get all repositories: %w", err) + } + for _, sr := range serverRepositories { + + repo := gitalypb.Repository{ + StorageName: sr.StorageName, + RelativePath: sr.RelativePath, + GlProjectPath: sr.GlProjectPath, + } + pipeline.Handle(ctx, backup.NewCreateCommand(manager, backup.CreateRequest{ + Server: sr.ServerInfo, + Repository: &repo, + VanityRepository: &repo, + Incremental: cmd.incremental, + BackupID: cmd.backupID, + }, + )) + } + + if _, err := pipeline.Done(); err != nil { + return fmt.Errorf("create: %w", err) + + } + return nil +} + +func (cmd *createSubcommand) pipeEntry(ctx context.Context, decoder *json.Decoder, manager backup.Strategy, pipeline *backup.Pipeline) error { + for { var sr serverRepository if err := decoder.Decode(&sr); errors.Is(err, io.EOF) { @@ -179,3 +229,72 @@ func (cmd *createSubcommand) run(ctx context.Context, logger log.Logger, stdin i } return nil } + +func getHashedStoragePaths() ([]storageInfo, error) { + var repoInfo []storageInfo + + configData, err := os.ReadFile(gitlabConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + config, err := toml.Load(string(configData)) + if err != nil { + return nil, fmt.Errorf("failed to parse TOML: %w", err) + } + + storagesRaw := config.Get("storage") + if storagesRaw == nil { + return nil, fmt.Errorf("storage section not found in config") + } + + storages, ok := storagesRaw.([]*toml.Tree) + if !ok { + return nil, fmt.Errorf("failed to parse storage section") + } + + for _, storage := range storages { + name, nameOK := storage.Get("name").(string) + path, pathOK := storage.Get("path").(string) + + if nameOK && pathOK { + repoInfo = append(repoInfo, storageInfo{ + StorageName: name, + RelativePath: filepath.Join(path, "@hashed"), // Ensure we scan @hashed + }) + } + } + + return repoInfo, nil +} + +func getAllRepositories() ([]serverRepository, error) { + repoHashesInfo, err := getHashedStoragePaths() + if err != nil { + return nil, err + } + + var serverRepositories []serverRepository + + for _, repoHashInfo := range repoHashesInfo { + + err := filepath.WalkDir(repoHashInfo.RelativePath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + if filepath.Ext(path) == ".git" && !d.IsDir() { + serverRepositories = append(serverRepositories, serverRepository{StorageName: repoHashInfo.StorageName, + RelativePath: path, + GlProjectPath: ""}) + } + return nil + }) + + if err != nil { + return nil, err + } + } + + return serverRepositories, nil +} -- GitLab From 94de20c79ad2129f8bdacb4694c394986df1ceba Mon Sep 17 00:00:00 2001 From: romongbale Date: Tue, 11 Feb 2025 09:43:05 -0500 Subject: [PATCH 2/2] feat: gitaly backup retrieve all repo storage in a node --- internal/cli/gitalybackup/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/gitalybackup/create.go b/internal/cli/gitalybackup/create.go index eacfd0a6fb..03db764e20 100644 --- a/internal/cli/gitalybackup/create.go +++ b/internal/cli/gitalybackup/create.go @@ -163,10 +163,10 @@ func (cmd *createSubcommand) run(ctx context.Context, logger log.Logger, stdin i return fmt.Errorf("create pipeline: %w", err) } - decoder := json.NewDecoder(stdin) if cmd.backupupAll { return cmd.pipeAll(ctx, manager, pipeline) } + decoder := json.NewDecoder(stdin) return cmd.pipeEntry(ctx, decoder, manager, pipeline) } -- GitLab