diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 9f32ee22e15d2b413b82f7e64db88d4a788b17b8..3abb42a08cb7cb7ee5a1068f20a86f1f0e7d0daf 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -7,8 +7,11 @@ import ( "errors" "fmt" "io" + "math" "strings" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -27,6 +30,19 @@ var ( ErrSkipped = errors.New("repository skipped") // ErrDoesntExist means that the data was not found. ErrDoesntExist = errors.New("doesn't exist") + + backupLatency = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gitaly_backup_latency_seconds", + Help: "Latency of a repository backup by phase", + }, + []string{"phase"}) + backupBundleSize = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "gitaly_backup_bundle_bytes", + Help: "Size of a Git bundle uploaded in a backup", + Buckets: prometheus.ExponentialBucketsRange(1, 10*math.Pow(1024, 3), 20), // up to 10GB + }) ) // Sink is an abstraction over the real storage used for storing/restoring backups. @@ -365,6 +381,9 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora } func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("bundle")) + defer timer.ObserveDuration() + var patterns io.Reader if len(step.PreviousRefPath) > 0 { // If there is a previous ref path, then we are creating an increment @@ -385,6 +404,7 @@ func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step return mgr.sink.GetWriter(ctx, step.BundlePath) }) defer func() { + backupBundleSize.Observe(float64(w.BytesWritten())) if err := w.Close(); err != nil && returnErr == nil { returnErr = fmt.Errorf("write bundle: %w", err) } @@ -477,6 +497,9 @@ func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path str } func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("custom_hooks")) + defer timer.ObserveDuration() + w := NewLazyWriter(func() (io.WriteCloser, error) { return mgr.sink.GetWriter(ctx, path) }) @@ -510,6 +533,9 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, pat // writeRefs writes the previously fetched list of refs in the same output // format as `git-show-ref(1)` func (mgr *Manager) writeRefs(ctx context.Context, path string, refs []git.Reference) (returnErr error) { + timer := prometheus.NewTimer(backupLatency.WithLabelValues("refs")) + defer timer.ObserveDuration() + w, err := mgr.sink.GetWriter(ctx, path) if err != nil { return fmt.Errorf("write refs: %w", err) diff --git a/internal/backup/lazy.go b/internal/backup/lazy.go index 7c38d5c6bc4fbf1b0b2f2c93d96660728e341cdf..a66b9062115ffd8d139057c742c1991410b3e600 100644 --- a/internal/backup/lazy.go +++ b/internal/backup/lazy.go @@ -8,8 +8,9 @@ import ( // Write. This means it will only create a file if there will be data written // to it. type LazyWriter struct { - create func() (io.WriteCloser, error) - w io.WriteCloser + create func() (io.WriteCloser, error) + w io.WriteCloser + bytesWritten int } // NewLazyWriter initializes a new LazyWriter. create is called on the first @@ -20,6 +21,12 @@ func NewLazyWriter(create func() (io.WriteCloser, error)) *LazyWriter { } } +// BytesWritten returns the total number of bytes written to the underlying +// WriteCloser. The count is never explicitly reset to 0. +func (w *LazyWriter) BytesWritten() int { + return w.bytesWritten +} + func (w *LazyWriter) Write(p []byte) (int, error) { if w.w == nil { var err error @@ -29,7 +36,9 @@ func (w *LazyWriter) Write(p []byte) (int, error) { } } - return w.w.Write(p) + n, err := w.w.Write(p) + w.bytesWritten += n + return n, err } // Close calls Close on the WriteCloser returned by Create, passing on any diff --git a/internal/backup/lazy_test.go b/internal/backup/lazy_test.go index 5f4ff514f7b8e19088816dde0a691827030165be..15cce107566045f8dad2edae1202ae8c6ad7e56c 100644 --- a/internal/backup/lazy_test.go +++ b/internal/backup/lazy_test.go @@ -36,6 +36,7 @@ func TestLazyWriter(t *testing.T) { _, err = io.Copy(w, iotest.OneByteReader(bytes.NewReader(expectedData))) require.NoError(t, err) require.FileExists(t, tempFilePath) + require.Equal(t, len(expectedData), w.BytesWritten()) data, err := os.ReadFile(tempFilePath) require.NoError(t, err) @@ -50,5 +51,6 @@ func TestLazyWriter(t *testing.T) { n, err := w.Write(make([]byte, 100)) require.Equal(t, 0, n) require.Equal(t, assert.AnError, err) + require.Equal(t, 0, w.BytesWritten()) }) }