From 120b7729242338870c844c093c26cc67ad2a4831 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 21 Nov 2023 10:39:50 +1100 Subject: [PATCH 1/4] backup: Add vars for server-side backup metrics Adds Prometheus observers for backup latency and bundle size to the backup package. These will later be used by the Manager to track metrics as a backup progresses. --- internal/backup/backup.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 9f32ee22e1..fa729d5235 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -7,8 +7,11 @@ import ( "errors" "fmt" "io" + "math" "strings" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -27,6 +30,19 @@ var ( ErrSkipped = errors.New("repository skipped") // ErrDoesntExist means that the data was not found. ErrDoesntExist = errors.New("doesn't exist") + + backupLatency = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gitaly_backup_latency_seconds", + Help: "Latency of a repository backup by phase", + }, + []string{"phase"}) + backupBundleSize = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "gitaly_backup_bundle_bytes", + Help: "Size of a Git bundle uploaded in a backup", + Buckets: prometheus.ExponentialBucketsRange(1, 10*math.Pow(1024, 3), 20), // up to 10GB + }) ) // Sink is an abstraction over the real storage used for storing/restoring backups. -- GitLab From c61688c5435332027dd00dcc401234991128e2e0 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 21 Nov 2023 10:47:08 +1100 Subject: [PATCH 2/4] backup: Track bytes written by LazyWriter Modifies the LazyWriter so the number of bytes written to the underlying WriteCloser is tracked. Adds a new public method to retrieve this value and updates existing tests to exercise this function. The number of bytes written will later be used to observe the bundle size metric added in the previous commit. --- internal/backup/lazy.go | 15 ++++++++++++--- internal/backup/lazy_test.go | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/internal/backup/lazy.go b/internal/backup/lazy.go index 7c38d5c6bc..a66b906211 100644 --- a/internal/backup/lazy.go +++ b/internal/backup/lazy.go @@ -8,8 +8,9 @@ import ( // Write. This means it will only create a file if there will be data written // to it. type LazyWriter struct { - create func() (io.WriteCloser, error) - w io.WriteCloser + create func() (io.WriteCloser, error) + w io.WriteCloser + bytesWritten int } // NewLazyWriter initializes a new LazyWriter. create is called on the first @@ -20,6 +21,12 @@ func NewLazyWriter(create func() (io.WriteCloser, error)) *LazyWriter { } } +// BytesWritten returns the total number of bytes written to the underlying +// WriteCloser. The count is never explicitly reset to 0. +func (w *LazyWriter) BytesWritten() int { + return w.bytesWritten +} + func (w *LazyWriter) Write(p []byte) (int, error) { if w.w == nil { var err error @@ -29,7 +36,9 @@ func (w *LazyWriter) Write(p []byte) (int, error) { } } - return w.w.Write(p) + n, err := w.w.Write(p) + w.bytesWritten += n + return n, err } // Close calls Close on the WriteCloser returned by Create, passing on any diff --git a/internal/backup/lazy_test.go b/internal/backup/lazy_test.go index 5f4ff514f7..15cce10756 100644 --- a/internal/backup/lazy_test.go +++ b/internal/backup/lazy_test.go @@ -36,6 +36,7 @@ func TestLazyWriter(t *testing.T) { _, err = io.Copy(w, iotest.OneByteReader(bytes.NewReader(expectedData))) require.NoError(t, err) require.FileExists(t, tempFilePath) + require.Equal(t, len(expectedData), w.BytesWritten()) data, err := os.ReadFile(tempFilePath) require.NoError(t, err) @@ -50,5 +51,6 @@ func TestLazyWriter(t *testing.T) { n, err := w.Write(make([]byte, 100)) require.Equal(t, 0, n) require.Equal(t, assert.AnError, err) + require.Equal(t, 0, w.BytesWritten()) }) } -- GitLab From 9e1adc6109b698e271f56f9e476a672acdf2cb9d Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 21 Nov 2023 11:32:05 +1100 Subject: [PATCH 3/4] backup: Instrument backup phases Adds instrumentation to each write phase of a repository backup so we can measure each phase separately. --- internal/backup/backup.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index fa729d5235..3bacb2d1ce 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -381,6 +381,9 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora } func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("bundle")) + defer timer.ObserveDuration() + var patterns io.Reader if len(step.PreviousRefPath) > 0 { // If there is a previous ref path, then we are creating an increment @@ -493,6 +496,9 @@ func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path str } func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("custom_hooks")) + defer timer.ObserveDuration() + w := NewLazyWriter(func() (io.WriteCloser, error) { return mgr.sink.GetWriter(ctx, path) }) @@ -526,6 +532,9 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, pat // writeRefs writes the previously fetched list of refs in the same output // format as `git-show-ref(1)` func (mgr *Manager) writeRefs(ctx context.Context, path string, refs []git.Reference) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("refs")) + defer timer.ObserveDuration() + w, err := mgr.sink.GetWriter(ctx, path) if err != nil { return fmt.Errorf("write refs: %w", err) -- GitLab From 102abfbebf12903bd412995a8b14f091f25f06d2 Mon Sep 17 00:00:00 2001 From: James Liu Date: Tue, 21 Nov 2023 11:38:49 +1100 Subject: [PATCH 4/4] backup: Instrument bundle write Adds instrumentation such that the total bundle size written to object storage is observed at the conclusion of the backup. --- internal/backup/backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 3bacb2d1ce..3abb42a08c 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -404,6 +404,7 @@ func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step return mgr.sink.GetWriter(ctx, step.BundlePath) }) defer func() { + backupBundleSize.Observe(float64(w.BytesWritten())) if err := w.Close(); err != nil && returnErr == nil { returnErr = fmt.Errorf("write bundle: %w", err) } -- GitLab