From 760bd3b5ef3248ae6692a385ace192fff7096704 Mon Sep 17 00:00:00 2001 From: James Fargher Date: Wed, 30 Aug 2023 16:44:29 +1200 Subject: [PATCH 1/3] benchmarking: Add benchmark for BackupRepository --- .../files/queries/BackupRepository/git.git.json | 17 +++++++++++++++++ .../roles/deploy/templates/all.yml.j2 | 6 ++++++ .../roles/gitaly/templates/config.toml.j2 | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 _support/benchmarking/roles/client/files/queries/BackupRepository/git.git.json diff --git a/_support/benchmarking/roles/client/files/queries/BackupRepository/git.git.json b/_support/benchmarking/roles/client/files/queries/BackupRepository/git.git.json new file mode 100644 index 0000000000..12c702619a --- /dev/null +++ b/_support/benchmarking/roles/client/files/queries/BackupRepository/git.git.json @@ -0,0 +1,17 @@ +{ + "repository": { + "storageName": "default", + "relativePath": "git.git", + "gitAlternateObjectDirectories": [], + "glRepository": "git", + "glProjectPath": "git/git" + }, + "vanityRepository": { + "storageName": "default", + "relativePath": "git.git", + "gitAlternateObjectDirectories": [], + "glRepository": "git", + "glProjectPath": "git/git" + }, + "backupId": "abc123" +} diff --git a/_support/benchmarking/roles/deploy/templates/all.yml.j2 b/_support/benchmarking/roles/deploy/templates/all.yml.j2 index d5c4c43ba9..e6eab95072 100644 --- a/_support/benchmarking/roles/deploy/templates/all.yml.j2 +++ b/_support/benchmarking/roles/deploy/templates/all.yml.j2 @@ -7,3 +7,9 @@ rpcs: proto: commit.proto repos: - git.git + - description: Create a backup + rpc: BackupRepository + service: gitaly.RepositoryService + proto: repository.proto + repos: + - git.git diff --git a/_support/benchmarking/roles/gitaly/templates/config.toml.j2 b/_support/benchmarking/roles/gitaly/templates/config.toml.j2 index 4e50eaa1db..ea18437f08 100644 --- a/_support/benchmarking/roles/gitaly/templates/config.toml.j2 +++ b/_support/benchmarking/roles/gitaly/templates/config.toml.j2 @@ -50,3 +50,5 @@ enabled = true dir = "/mnt/git-repositories/+gitaly/PackObjectsCache" max_age = '5m' +[backup] +go_cloud_url = "/var/opt/gitaly/backups" -- GitLab From 6dc646f9fa0de6e676f6a77cbbfca2006f78fa11 Mon Sep 17 00:00:00 2001 From: James Fargher Date: Thu, 14 Sep 2023 15:25:08 +1200 Subject: [PATCH 2/3] backup: Stop using LazyWriter on bundles --- internal/backup/backup.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 267ec5f999..07d3ad73b1 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -354,9 +354,11 @@ func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step } }() - w := NewLazyWriter(func() (io.WriteCloser, error) { - return mgr.sink.GetWriter(ctx, step.BundlePath) - }) + w, err := mgr.sink.GetWriter(ctx, step.BundlePath) + if err != nil { + return fmt.Errorf("write bundle: %w", err) + } + defer func() { if err := w.Close(); err != nil && returnErr == nil { returnErr = fmt.Errorf("write bundle: %w", err) -- GitLab From 1109a5c2dde4886515243f7b2a9feeeca6aef394 Mon Sep 17 00:00:00 2001 From: James Fargher Date: Thu, 14 Sep 2023 16:25:59 +1200 Subject: [PATCH 3/3] backup: Buffer small writes --- internal/backup/backup.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 07d3ad73b1..cad8cefd93 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -1,6 +1,7 @@ package backup import ( + "bufio" "bytes" "context" "errors" @@ -392,6 +393,7 @@ func (mgr *Manager) negatedKnownRefs(ctx context.Context, step *Step) (io.ReadCl } defer reader.Close() + buf := bufio.NewWriter(w) d := git.NewShowRefDecoder(reader) for { var ref git.Reference @@ -403,11 +405,16 @@ func (mgr *Manager) negatedKnownRefs(ctx context.Context, step *Step) (io.ReadCl return } - if _, err := fmt.Fprintf(w, "^%s\n", ref.Target); err != nil { + if _, err := fmt.Fprintf(buf, "^%s\n", ref.Target); err != nil { _ = w.CloseWithError(err) return } } + + if err := buf.Flush(); err != nil { + _ = w.CloseWithError(err) + return + } }() return r, nil @@ -495,13 +502,19 @@ func (mgr *Manager) writeRefs(ctx context.Context, path string, refs []git.Refer } }() + buf := bufio.NewWriter(w) + for _, ref := range refs { - _, err = fmt.Fprintf(w, "%s %s\n", ref.Target, ref.Name) + _, err = fmt.Fprintf(buf, "%s %s\n", ref.Target, ref.Name) if err != nil { return fmt.Errorf("write refs: %w", err) } } + if err := buf.Flush(); err != nil { + return fmt.Errorf("write refs: %w", err) + } + return nil } -- GitLab