From 21dd57f017a0de5787c4d83ed2ded145ff258b97 Mon Sep 17 00:00:00 2001 From: syedsaifalialvi Date: Tue, 2 Dec 2025 10:50:34 +0530 Subject: [PATCH] Store metadata when offloading a repo This will store the following metadata, when offloading the repository - storage-name - storage-path - relative-path - partition-id --- .../transaction_manager_housekeeping.go | 11 ++++++++- internal/offloading/sink.go | 3 ++- internal/offloading/sink_test.go | 23 +++++++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping.go index 94a7706fc3..f058b5af2f 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping.go @@ -916,8 +916,17 @@ func (mgr *TransactionManager) prepareOffloading(ctx context.Context, transactio } } }() + + // Prepare metadata for offloading + metadataMap := map[string]string{ + "storage-name": originalRepo.GetStorageName(), + "storage-path": mgr.storagePath, + "relative-path": originalRepo.GetRelativePath(), + "partition-id": mgr.partitionID.String(), + } + for file := range packFilesToUpload { - if err := mgr.offloadingSink.Upload(ctx, filepath.Join(filterToDir, file), cfg.Prefix); err != nil { + if err := mgr.offloadingSink.Upload(ctx, filepath.Join(filterToDir, file), cfg.Prefix, metadataMap); err != nil { return errors.Join(errOffloadingObjectUpload, err) } uploadedPackFiles = append(uploadedPackFiles, file) diff --git a/internal/offloading/sink.go b/internal/offloading/sink.go index a500de24d0..57a4306e1b 100644 --- a/internal/offloading/sink.go +++ b/internal/offloading/sink.go @@ -86,7 +86,7 @@ func NewSink(bucket Bucket, options ...SinkOption) (*Sink, error) { // Upload uploads a file located at fullFilePath to the bucket under the specified prefix. // The fullFilePath include the file name, e.g. /tmp/foo.txt. -func (r *Sink) Upload(ctx context.Context, fullFilePath string, prefix string) (returnErr error) { +func (r *Sink) Upload(ctx context.Context, fullFilePath string, prefix string, metadata map[string]string) (returnErr error) { ctx, cancel := context.WithTimeout(ctx, r.overallTimeout) defer cancel() @@ -110,6 +110,7 @@ func (r *Sink) Upload(ctx context.Context, fullFilePath string, prefix string) ( // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#other CacheControl: "no-store, no-transform", ContentType: "application/octet-stream", + Metadata: metadata, }) }); err != nil { return fmt.Errorf("upload object %q: %w", objectKey, err) diff --git a/internal/offloading/sink_test.go b/internal/offloading/sink_test.go index 64232a4549..03f76734e3 100644 --- a/internal/offloading/sink_test.go +++ b/internal/offloading/sink_test.go @@ -113,7 +113,7 @@ func TestSink_Upload(t *testing.T) { expectedErrored map[string]error }{ { - desc: "Upload objects", + desc: "Upload objects with metadata", prefix: "jerry", filesToUpload: map[string]string{ "C-131": "I am Mr. Frundles", @@ -172,13 +172,22 @@ func TestSink_Upload(t *testing.T) { } objectName := fileName - err := sink.Upload(ctx, filepath.Join(localDir, objectName), tc.prefix) + testMetadata := map[string]string{ + "storage-name": "default", + "storage-path": "/var/opt/gitlab/git-data", + "relative-path": "gitlab/gitaly.git", + "partition-id": "partition-1", + } + + err := sink.Upload(ctx, filepath.Join(localDir, objectName), tc.prefix, testMetadata) if err == nil { objKey := tc.prefix + "/" + objectName attr, err := sink.bucket.Attributes(ctx, objKey) require.NoError(t, err) require.Equal(t, attr.CacheControl, "no-store, no-transform") require.Equal(t, attr.ContentType, "application/octet-stream") + require.NotNil(t, attr.Metadata, "metadata should be stored with the object") + require.Equal(t, testMetadata, attr.Metadata, "stored metadata should match uploaded metadata for object %s", objectName) var builder strings.Builder err = sink.bucket.Download(ctx, objKey, &builder, nil) @@ -202,9 +211,9 @@ func TestSink_Upload(t *testing.T) { sink := setupEmptyLocalBucket(t) - err = sink.Upload(ctx, filepath.Join(localDirLoser, "i_am_key"), "some/prefix") + err = sink.Upload(ctx, filepath.Join(localDirLoser, "i_am_key"), "some/prefix", nil) require.NoError(t, err) - err = sink.Upload(ctx, filepath.Join(localDirWinner, "i_am_key"), "some/prefix") + err = sink.Upload(ctx, filepath.Join(localDirWinner, "i_am_key"), "some/prefix", nil) require.NoError(t, err) var builder strings.Builder @@ -218,7 +227,7 @@ func TestSink_Upload(t *testing.T) { sink := setupEmptyLocalBucket(t) defer closeBucket(t, sink) - err := sink.Upload(ctx, "", "some/prefix") + err := sink.Upload(ctx, "", "some/prefix", nil) require.Error(t, err) }) } @@ -327,7 +336,7 @@ func TestSink_Upload_Timeout_Cancellation_And_Retry(t *testing.T) { err = os.WriteFile(filepath.Join(localDir, tc.objectName), []byte("Go long!"), mode.File) require.NoError(t, err) - err = sink.Upload(ctx, filepath.Join(localDir, tc.objectName), prefix) + err = sink.Upload(ctx, filepath.Join(localDir, tc.objectName), prefix, nil) if tc.expectedError != nil { require.ErrorIs(t, err, errSimulationCanceled) @@ -374,7 +383,7 @@ func TestSink_Upload_Timeout_Cancellation_And_Retry(t *testing.T) { errCh := make(chan error) go func() { - errCh <- sink.Upload(ctx, filepath.Join(localDir, objectName), prefix) + errCh <- sink.Upload(ctx, filepath.Join(localDir, objectName), prefix, nil) }() // Add a small delay before cancellation to ensure operation has started -- GitLab