From 93c098e3effd5a6214785616e1e1b0b25850504c Mon Sep 17 00:00:00 2001
From: Mustafa Bayar <mbayar@gitlab.com>
Date: Thu, 24 Oct 2024 11:33:52 +0300
Subject: [PATCH 1/4] protobuf: Create partition backup manifest RPC

Create a new RPC that will be used to create/update manifests for
partition backups. We also need to modify mockPartitionServiceClient
in partition_backup_test to support the new method.
---
 internal/backup/partition_backup_test.go |   4 +
 proto/go/gitalypb/partition.pb.go        | 347 ++++++++++++++++++-----
 proto/go/gitalypb/partition_grpc.pb.go   |  44 ++-
 proto/partition.proto                    |  33 +++
 4 files changed, 359 insertions(+), 69 deletions(-)

diff --git a/internal/backup/partition_backup_test.go b/internal/backup/partition_backup_test.go
index 44a3cba1b5..2c4b37a552 100644
--- a/internal/backup/partition_backup_test.go
+++ b/internal/backup/partition_backup_test.go
@@ -283,3 +283,7 @@ func (m *mockPartitionServiceClient) BackupPartition(ctx context.Context, req *g
 
 	return &gitalypb.BackupPartitionResponse{}, nil
 }
+
+func (m *mockPartitionServiceClient) CreatePartitionBackupManifest(ctx context.Context, req *gitalypb.CreatePartitionBackupManifestRequest, opts ...grpc.CallOption) (*gitalypb.CreatePartitionBackupManifestResponse, error) {
+	return &gitalypb.CreatePartitionBackupManifestResponse{}, nil
+}
diff --git a/proto/go/gitalypb/partition.pb.go b/proto/go/gitalypb/partition.pb.go
index 29ee8b493a..7821d71f16 100644
--- a/proto/go/gitalypb/partition.pb.go
+++ b/proto/go/gitalypb/partition.pb.go
@@ -9,6 +9,7 @@ package gitalypb
 import (
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -228,56 +229,259 @@ func (x *ListPartitionsResponse) GetPaginationCursor() *PaginationCursor {
 	return nil
 }
 
+// CreatePartitionBackupManifestRequest is a request for the CreatePartitionBackupManifest RPC.
+type CreatePartitionBackupManifestRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// storage_name is the name of the storage containing the archived partitions.
+	StorageName string `protobuf:"bytes,1,opt,name=storage_name,json=storageName,proto3" json:"storage_name,omitempty"`
+	// timestamp is the global time of the backup.
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	// partitions contains the list of partitions that are archived.
+	Partitions []*PartitionBackupInfo `protobuf:"bytes,3,rep,name=partitions,proto3" json:"partitions,omitempty"`
+}
+
+func (x *CreatePartitionBackupManifestRequest) Reset() {
+	*x = CreatePartitionBackupManifestRequest{}
+	mi := &file_partition_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *CreatePartitionBackupManifestRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreatePartitionBackupManifestRequest) ProtoMessage() {}
+
+func (x *CreatePartitionBackupManifestRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_partition_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreatePartitionBackupManifestRequest.ProtoReflect.Descriptor instead.
+func (*CreatePartitionBackupManifestRequest) Descriptor() ([]byte, []int) {
+	return file_partition_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *CreatePartitionBackupManifestRequest) GetStorageName() string {
+	if x != nil {
+		return x.StorageName
+	}
+	return ""
+}
+
+func (x *CreatePartitionBackupManifestRequest) GetTimestamp() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Timestamp
+	}
+	return nil
+}
+
+func (x *CreatePartitionBackupManifestRequest) GetPartitions() []*PartitionBackupInfo {
+	if x != nil {
+		return x.Partitions
+	}
+	return nil
+}
+
+// PartitionBackupInfo metadata for the partition that is archived
+type PartitionBackupInfo struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// timestamp is the time of the partition backup.
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	// partition_id is the identifier of the partition.
+	PartitionId string `protobuf:"bytes,2,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"`
+	// backup_relative_path contains the relative path to the backup.
+	BackupRelativePath string `protobuf:"bytes,3,opt,name=backup_relative_path,json=backupRelativePath,proto3" json:"backup_relative_path,omitempty"`
+}
+
+func (x *PartitionBackupInfo) Reset() {
+	*x = PartitionBackupInfo{}
+	mi := &file_partition_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PartitionBackupInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PartitionBackupInfo) ProtoMessage() {}
+
+func (x *PartitionBackupInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_partition_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PartitionBackupInfo.ProtoReflect.Descriptor instead.
+func (*PartitionBackupInfo) Descriptor() ([]byte, []int) {
+	return file_partition_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *PartitionBackupInfo) GetTimestamp() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Timestamp
+	}
+	return nil
+}
+
+func (x *PartitionBackupInfo) GetPartitionId() string {
+	if x != nil {
+		return x.PartitionId
+	}
+	return ""
+}
+
+func (x *PartitionBackupInfo) GetBackupRelativePath() string {
+	if x != nil {
+		return x.BackupRelativePath
+	}
+	return ""
+}
+
+// CreatePartitionBackupManifestResponse is a response for the CreatePartitionBackupManifest RPC.
+type CreatePartitionBackupManifestResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *CreatePartitionBackupManifestResponse) Reset() {
+	*x = CreatePartitionBackupManifestResponse{}
+	mi := &file_partition_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *CreatePartitionBackupManifestResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreatePartitionBackupManifestResponse) ProtoMessage() {}
+
+func (x *CreatePartitionBackupManifestResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_partition_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreatePartitionBackupManifestResponse.ProtoReflect.Descriptor instead.
+func (*CreatePartitionBackupManifestResponse) Descriptor() ([]byte, []int) {
+	return file_partition_proto_rawDescGZIP(), []int{6}
+}
+
 var File_partition_proto protoreflect.FileDescriptor
 
 var file_partition_proto_rawDesc = []byte{
 	0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x12, 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x0a, 0x6c, 0x69, 0x6e, 0x74, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0c, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x22, 0x6a, 0x0a, 0x16, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72,
-	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a,
-	0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61,
-	0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74,
-	0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0xa8, 0xc6,
-	0x2c, 0x01, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22,
-	0x19, 0x0a, 0x17, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8a, 0x01, 0x0a, 0x15, 0x4c,
-	0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f,
-	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6, 0x2c, 0x01,
-	0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x48, 0x0a,
-	0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x72, 0x61,
-	0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c,
-	0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61,
-	0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74,
-	0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x12, 0x31, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e,
-	0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x45, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67, 0x69,
-	0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x32, 0xcb, 0x01, 0x0a,
-	0x10, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
-	0x65, 0x12, 0x5c, 0x0a, 0x0f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61,
-	0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61,
-	0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x03, 0x12,
-	0x59, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
-	0x73, 0x12, 0x1d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50,
-	0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61,
-	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69,
-	0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d,
-	0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x12, 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73,
+	0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0a, 0x6c, 0x69, 0x6e, 0x74,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0c, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6a, 0x0a, 0x16, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27,
+	0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72,
+	0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69,
+	0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0xa8,
+	0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64,
+	0x22, 0x19, 0x0a, 0x17, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74,
+	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8a, 0x01, 0x0a, 0x15,
+	0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+	0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6, 0x2c,
+	0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x48,
+	0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x72,
+	0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x69, 0x74, 0x61,
+	0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72,
+	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73,
+	0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
+	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79,
+	0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74,
+	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x45, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67,
+	0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, 0xc6, 0x01,
+	0x0a, 0x24, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67,
+	0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6,
+	0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12,
+	0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09,
+	0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x61, 0x72,
+	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e,
+	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
+	0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74,
+	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa4, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69,
+	0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38,
+	0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74,
+	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74,
+	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
+	0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x62,
+	0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70,
+	0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x61, 0x63, 0x6b, 0x75,
+	0x70, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x22, 0x27, 0x0a,
+	0x25, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
+	0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd4, 0x02, 0x0a, 0x10, 0x50, 0x61, 0x72, 0x74, 0x69,
+	0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x0f, 0x42,
+	0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e,
+	0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f,
+	0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x03, 0x12, 0x59, 0x0a, 0x0e, 0x4c, 0x69, 0x73,
+	0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x67, 0x69,
+	0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
+	0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x69, 0x74,
+	0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04,
+	0x08, 0x02, 0x10, 0x02, 0x12, 0x86, 0x01, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50,
+	0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61,
+	0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e,
+	0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42,
+	0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x43, 0x72,
+	0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63,
+	0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x42, 0x34, 0x5a,
+	0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c,
+	0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31,
+	0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c,
+	0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -292,29 +496,38 @@ func file_partition_proto_rawDescGZIP() []byte {
 	return file_partition_proto_rawDescData
 }
 
-var file_partition_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_partition_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
 var file_partition_proto_goTypes = []any{
-	(*BackupPartitionRequest)(nil),  // 0: gitaly.BackupPartitionRequest
-	(*BackupPartitionResponse)(nil), // 1: gitaly.BackupPartitionResponse
-	(*ListPartitionsRequest)(nil),   // 2: gitaly.ListPartitionsRequest
-	(*ListPartitionsResponse)(nil),  // 3: gitaly.ListPartitionsResponse
-	(*PaginationParameter)(nil),     // 4: gitaly.PaginationParameter
-	(*Partition)(nil),               // 5: gitaly.Partition
-	(*PaginationCursor)(nil),        // 6: gitaly.PaginationCursor
+	(*BackupPartitionRequest)(nil),                // 0: gitaly.BackupPartitionRequest
+	(*BackupPartitionResponse)(nil),               // 1: gitaly.BackupPartitionResponse
+	(*ListPartitionsRequest)(nil),                 // 2: gitaly.ListPartitionsRequest
+	(*ListPartitionsResponse)(nil),                // 3: gitaly.ListPartitionsResponse
+	(*CreatePartitionBackupManifestRequest)(nil),  // 4: gitaly.CreatePartitionBackupManifestRequest
+	(*PartitionBackupInfo)(nil),                   // 5: gitaly.PartitionBackupInfo
+	(*CreatePartitionBackupManifestResponse)(nil), // 6: gitaly.CreatePartitionBackupManifestResponse
+	(*PaginationParameter)(nil),                   // 7: gitaly.PaginationParameter
+	(*Partition)(nil),                             // 8: gitaly.Partition
+	(*PaginationCursor)(nil),                      // 9: gitaly.PaginationCursor
+	(*timestamppb.Timestamp)(nil),                 // 10: google.protobuf.Timestamp
 }
 var file_partition_proto_depIdxs = []int32{
-	4, // 0: gitaly.ListPartitionsRequest.pagination_params:type_name -> gitaly.PaginationParameter
-	5, // 1: gitaly.ListPartitionsResponse.partitions:type_name -> gitaly.Partition
-	6, // 2: gitaly.ListPartitionsResponse.pagination_cursor:type_name -> gitaly.PaginationCursor
-	0, // 3: gitaly.PartitionService.BackupPartition:input_type -> gitaly.BackupPartitionRequest
-	2, // 4: gitaly.PartitionService.ListPartitions:input_type -> gitaly.ListPartitionsRequest
-	1, // 5: gitaly.PartitionService.BackupPartition:output_type -> gitaly.BackupPartitionResponse
-	3, // 6: gitaly.PartitionService.ListPartitions:output_type -> gitaly.ListPartitionsResponse
-	5, // [5:7] is the sub-list for method output_type
-	3, // [3:5] is the sub-list for method input_type
-	3, // [3:3] is the sub-list for extension type_name
-	3, // [3:3] is the sub-list for extension extendee
-	0, // [0:3] is the sub-list for field type_name
+	7,  // 0: gitaly.ListPartitionsRequest.pagination_params:type_name -> gitaly.PaginationParameter
+	8,  // 1: gitaly.ListPartitionsResponse.partitions:type_name -> gitaly.Partition
+	9,  // 2: gitaly.ListPartitionsResponse.pagination_cursor:type_name -> gitaly.PaginationCursor
+	10, // 3: gitaly.CreatePartitionBackupManifestRequest.timestamp:type_name -> google.protobuf.Timestamp
+	5,  // 4: gitaly.CreatePartitionBackupManifestRequest.partitions:type_name -> gitaly.PartitionBackupInfo
+	10, // 5: gitaly.PartitionBackupInfo.timestamp:type_name -> google.protobuf.Timestamp
+	0,  // 6: gitaly.PartitionService.BackupPartition:input_type -> gitaly.BackupPartitionRequest
+	2,  // 7: gitaly.PartitionService.ListPartitions:input_type -> gitaly.ListPartitionsRequest
+	4,  // 8: gitaly.PartitionService.CreatePartitionBackupManifest:input_type -> gitaly.CreatePartitionBackupManifestRequest
+	1,  // 9: gitaly.PartitionService.BackupPartition:output_type -> gitaly.BackupPartitionResponse
+	3,  // 10: gitaly.PartitionService.ListPartitions:output_type -> gitaly.ListPartitionsResponse
+	6,  // 11: gitaly.PartitionService.CreatePartitionBackupManifest:output_type -> gitaly.CreatePartitionBackupManifestResponse
+	9,  // [9:12] is the sub-list for method output_type
+	6,  // [6:9] is the sub-list for method input_type
+	6,  // [6:6] is the sub-list for extension type_name
+	6,  // [6:6] is the sub-list for extension extendee
+	0,  // [0:6] is the sub-list for field type_name
 }
 
 func init() { file_partition_proto_init() }
@@ -330,7 +543,7 @@ func file_partition_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_partition_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   4,
+			NumMessages:   7,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
diff --git a/proto/go/gitalypb/partition_grpc.pb.go b/proto/go/gitalypb/partition_grpc.pb.go
index a5d67cc187..bb0fb45866 100644
--- a/proto/go/gitalypb/partition_grpc.pb.go
+++ b/proto/go/gitalypb/partition_grpc.pb.go
@@ -19,8 +19,9 @@ import (
 const _ = grpc.SupportPackageIsVersion9
 
 const (
-	PartitionService_BackupPartition_FullMethodName = "/gitaly.PartitionService/BackupPartition"
-	PartitionService_ListPartitions_FullMethodName  = "/gitaly.PartitionService/ListPartitions"
+	PartitionService_BackupPartition_FullMethodName               = "/gitaly.PartitionService/BackupPartition"
+	PartitionService_ListPartitions_FullMethodName                = "/gitaly.PartitionService/ListPartitions"
+	PartitionService_CreatePartitionBackupManifest_FullMethodName = "/gitaly.PartitionService/CreatePartitionBackupManifest"
 )
 
 // PartitionServiceClient is the client API for PartitionService service.
@@ -34,6 +35,8 @@ type PartitionServiceClient interface {
 	BackupPartition(ctx context.Context, in *BackupPartitionRequest, opts ...grpc.CallOption) (*BackupPartitionResponse, error)
 	// ListPartitions lists partitions present in the storage.
 	ListPartitions(ctx context.Context, in *ListPartitionsRequest, opts ...grpc.CallOption) (*ListPartitionsResponse, error)
+	// CreatePartitionBackupManifest creates the manifest for locating backups.
+	CreatePartitionBackupManifest(ctx context.Context, in *CreatePartitionBackupManifestRequest, opts ...grpc.CallOption) (*CreatePartitionBackupManifestResponse, error)
 }
 
 type partitionServiceClient struct {
@@ -64,6 +67,16 @@ func (c *partitionServiceClient) ListPartitions(ctx context.Context, in *ListPar
 	return out, nil
 }
 
+func (c *partitionServiceClient) CreatePartitionBackupManifest(ctx context.Context, in *CreatePartitionBackupManifestRequest, opts ...grpc.CallOption) (*CreatePartitionBackupManifestResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CreatePartitionBackupManifestResponse)
+	err := c.cc.Invoke(ctx, PartitionService_CreatePartitionBackupManifest_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // PartitionServiceServer is the server API for PartitionService service.
 // All implementations must embed UnimplementedPartitionServiceServer
 // for forward compatibility.
@@ -75,6 +88,8 @@ type PartitionServiceServer interface {
 	BackupPartition(context.Context, *BackupPartitionRequest) (*BackupPartitionResponse, error)
 	// ListPartitions lists partitions present in the storage.
 	ListPartitions(context.Context, *ListPartitionsRequest) (*ListPartitionsResponse, error)
+	// CreatePartitionBackupManifest creates the manifest for locating backups.
+	CreatePartitionBackupManifest(context.Context, *CreatePartitionBackupManifestRequest) (*CreatePartitionBackupManifestResponse, error)
 	mustEmbedUnimplementedPartitionServiceServer()
 }
 
@@ -91,6 +106,9 @@ func (UnimplementedPartitionServiceServer) BackupPartition(context.Context, *Bac
 func (UnimplementedPartitionServiceServer) ListPartitions(context.Context, *ListPartitionsRequest) (*ListPartitionsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ListPartitions not implemented")
 }
+func (UnimplementedPartitionServiceServer) CreatePartitionBackupManifest(context.Context, *CreatePartitionBackupManifestRequest) (*CreatePartitionBackupManifestResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreatePartitionBackupManifest not implemented")
+}
 func (UnimplementedPartitionServiceServer) mustEmbedUnimplementedPartitionServiceServer() {}
 func (UnimplementedPartitionServiceServer) testEmbeddedByValue()                          {}
 
@@ -148,6 +166,24 @@ func _PartitionService_ListPartitions_Handler(srv interface{}, ctx context.Conte
 	return interceptor(ctx, in, info, handler)
 }
 
+func _PartitionService_CreatePartitionBackupManifest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreatePartitionBackupManifestRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(PartitionServiceServer).CreatePartitionBackupManifest(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: PartitionService_CreatePartitionBackupManifest_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(PartitionServiceServer).CreatePartitionBackupManifest(ctx, req.(*CreatePartitionBackupManifestRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // PartitionService_ServiceDesc is the grpc.ServiceDesc for PartitionService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -163,6 +199,10 @@ var PartitionService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ListPartitions",
 			Handler:    _PartitionService_ListPartitions_Handler,
 		},
+		{
+			MethodName: "CreatePartitionBackupManifest",
+			Handler:    _PartitionService_CreatePartitionBackupManifest_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "partition.proto",
diff --git a/proto/partition.proto b/proto/partition.proto
index 151814353e..f17aa7cafc 100644
--- a/proto/partition.proto
+++ b/proto/partition.proto
@@ -2,6 +2,7 @@ syntax = "proto3";
 
 package gitaly;
 
+import "google/protobuf/timestamp.proto";
 import "lint.proto";
 import "shared.proto";
 
@@ -27,6 +28,14 @@ service PartitionService {
     };
   }
 
+  // CreatePartitionBackupManifest creates the manifest for locating backups.
+  rpc CreatePartitionBackupManifest(CreatePartitionBackupManifestRequest) returns (CreatePartitionBackupManifestResponse) {
+    option (op_type) = {
+      op:          ACCESSOR
+      scope_level: STORAGE
+    };
+  }
+
 }
 
 // BackupPartitionRequest is a request for the BackupPartition RPC.
@@ -60,3 +69,27 @@ message ListPartitionsResponse {
   // documentation for further info.
   PaginationCursor pagination_cursor = 2;
 }
+
+// CreatePartitionBackupManifestRequest is a request for the CreatePartitionBackupManifest RPC.
+message CreatePartitionBackupManifestRequest {
+  // storage_name is the name of the storage containing the archived partitions.
+  string storage_name = 1 [(gitaly.storage)=true];
+  // timestamp is the global time of the backup.
+  google.protobuf.Timestamp timestamp = 2;
+  // partitions contains the list of partitions that are archived.
+  repeated PartitionBackupInfo partitions = 3;
+}
+
+// PartitionBackupInfo metadata for the partition that is archived
+message PartitionBackupInfo {
+  // timestamp is the time of the partition backup.
+  google.protobuf.Timestamp timestamp = 1;
+  // partition_id is the identifier of the partition.
+  string partition_id = 2;
+  // backup_relative_path contains the relative path to the backup.
+  string backup_relative_path = 3;
+}
+
+// CreatePartitionBackupManifestResponse is a response for the CreatePartitionBackupManifest RPC.
+message CreatePartitionBackupManifestResponse {
+}
-- 
GitLab


From ed255cbd1ea4275d2f74b3178e7c5f1f1db63396 Mon Sep 17 00:00:00 2001
From: Mustafa Bayar <mbayar@gitlab.com>
Date: Thu, 24 Oct 2024 11:36:11 +0300
Subject: [PATCH 2/4] partition: Implement CreatePartitionBackupManifest RPC
 server

Implementation of CreatePartitionBackupManifest RPC that will receive
the partition backup information and upload it to the configured backup
sink to be able to locate the correct backups during restore.
---
 .../create_partition_backup_manifest.go       | 79 +++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 internal/gitaly/service/partition/create_partition_backup_manifest.go

diff --git a/internal/gitaly/service/partition/create_partition_backup_manifest.go b/internal/gitaly/service/partition/create_partition_backup_manifest.go
new file mode 100644
index 0000000000..e07b635179
--- /dev/null
+++ b/internal/gitaly/service/partition/create_partition_backup_manifest.go
@@ -0,0 +1,79 @@
+package partition
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
+	"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+)
+
+// CreatePartitionBackupManifest creates the manifest for locating backups.
+func (s *server) CreatePartitionBackupManifest(ctx context.Context, in *gitalypb.CreatePartitionBackupManifestRequest) (_ *gitalypb.CreatePartitionBackupManifestResponse, returnErr error) {
+	if s.backupSink == nil {
+		return nil, structerr.NewFailedPrecondition("create partition backup manifest: server-side backups are not configured")
+	}
+
+	manifestRelativePath := filepath.Join("partition-manifests", in.GetStorageName(), in.GetTimestamp().AsTime().String()+".json")
+
+	exists, err := s.backupSink.Exists(ctx, manifestRelativePath)
+	if err != nil || exists {
+		return nil, fmt.Errorf("manifest exists: %w", err)
+	}
+
+	// Create a new context to abort the write on failure.
+	writeCtx, cancelWrite := context.WithCancel(ctx)
+	defer cancelWrite()
+
+	w, err := s.backupSink.GetWriter(writeCtx, manifestRelativePath)
+	if err != nil {
+		return nil, fmt.Errorf("get backup writer: %w", err)
+	}
+	defer func() {
+		if returnErr != nil {
+			// End the context before calling Close to ensure we don't persist the failed
+			// write to object storage.
+			cancelWrite()
+		}
+		if err := w.Close(); err != nil && returnErr == nil {
+			returnErr = fmt.Errorf("close manifest writer: %w", err)
+		}
+	}()
+
+	encoder := json.NewEncoder(w)
+
+	// Write the global timestamp entry
+	if err := encoder.Encode(struct {
+		Timestamp time.Time `json:"timestamp"`
+	}{
+		Timestamp: in.GetTimestamp().AsTime(),
+	}); err != nil {
+		return nil, fmt.Errorf("encode manifest timestamp: %w", err)
+	}
+
+	// Process each partition backup
+	for _, partition := range in.GetPartitions() {
+		entry := ManifestEntry{
+			PartitionID:        partition.GetPartitionId(),
+			BackupRelativePath: partition.GetBackupRelativePath(),
+			Timestamp:          partition.GetTimestamp().AsTime(),
+		}
+
+		// Encode and write the entry
+		if err := encoder.Encode(entry); err != nil {
+			return nil, fmt.Errorf("encode manifest entry: %w", err)
+		}
+	}
+
+	return &gitalypb.CreatePartitionBackupManifestResponse{}, nil
+}
+
+// ManifestEntry is the representation of the backup manifest entry.
+type ManifestEntry struct {
+	PartitionID        string    `json:"partition_id"`
+	BackupRelativePath string    `json:"backup_relative_path"`
+	Timestamp          time.Time `json:"timestamp"`
+}
-- 
GitLab


From 515018dcca198fa33138bf743d22030b33faf9eb Mon Sep 17 00:00:00 2001
From: Mustafa Bayar <mbayar@gitlab.com>
Date: Thu, 24 Oct 2024 11:49:06 +0300
Subject: [PATCH 3/4] partition: Modify BackupPartition response

Return relative backup path in the BackupPartition RPC response
which will be used to create backup manifest.
With this, we also need to change the way we handle the already
exists backup case. We count them as successful backups in the
PartitionBackupManager, but while it is returning error, it can't
return the backup relative path response. Therefore I am changing
it's implementation to not return anymore and have it a successful
flow by default.
---
 .../service/partition/backup_partition.go     |  10 +-
 .../partition/backup_partition_test.go        |  19 ++-
 proto/go/gitalypb/partition.pb.go             | 151 ++++++++++--------
 proto/partition.proto                         |   2 +
 4 files changed, 103 insertions(+), 79 deletions(-)

diff --git a/internal/gitaly/service/partition/backup_partition.go b/internal/gitaly/service/partition/backup_partition.go
index c6dff46e72..e94ba84e19 100644
--- a/internal/gitaly/service/partition/backup_partition.go
+++ b/internal/gitaly/service/partition/backup_partition.go
@@ -27,14 +27,16 @@ func (s *server) BackupPartition(ctx context.Context, in *gitalypb.BackupPartiti
 		return nil, structerr.NewInternal("backup partition: transaction not initialized")
 	}
 
-	backupRelativePath := filepath.Join(in.GetStorageName(), in.GetPartitionId(), lsn+".tar")
+	backupRelativePath := filepath.Join("partition-backups", in.GetStorageName(), in.GetPartitionId(), lsn+".tar")
 
 	exists, err := s.backupSink.Exists(ctx, backupRelativePath)
 	if err != nil {
 		return nil, fmt.Errorf("backup exists: %w", err)
 	}
 	if exists {
-		return nil, structerr.NewAlreadyExists("there is an up-to-date backup for the given partition")
+		return &gitalypb.BackupPartitionResponse{
+			BackupRelativePath: backupRelativePath,
+		}, nil
 	}
 
 	// Create a new context to abort the write on failure.
@@ -60,5 +62,7 @@ func (s *server) BackupPartition(ctx context.Context, in *gitalypb.BackupPartiti
 		return nil, fmt.Errorf("write tarball: %w", err)
 	}
 
-	return &gitalypb.BackupPartitionResponse{}, nil
+	return &gitalypb.BackupPartitionResponse{
+		BackupRelativePath: backupRelativePath,
+	}, nil
 }
diff --git a/internal/gitaly/service/partition/backup_partition_test.go b/internal/gitaly/service/partition/backup_partition_test.go
index 0d6dbc08e9..3d79d44cc2 100644
--- a/internal/gitaly/service/partition/backup_partition_test.go
+++ b/internal/gitaly/service/partition/backup_partition_test.go
@@ -134,10 +134,12 @@ func TestBackupPartition(t *testing.T) {
 			}
 
 			require.NoError(t, err)
-			testhelper.ProtoEqual(t, &gitalypb.BackupPartitionResponse{}, resp)
+			testhelper.ProtoEqual(t, &gitalypb.BackupPartitionResponse{
+				BackupRelativePath: filepath.Join("partition-backups", "default", "2", "0000000000002.tar"),
+			}, resp)
 
 			lsn := storage.LSN(2)
-			tarPath := filepath.Join(backupRoot, data.storageName, data.partitionID, lsn.String()) + ".tar"
+			tarPath := filepath.Join(backupRoot, "partition-backups", data.storageName, data.partitionID, lsn.String()) + ".tar"
 			tar, err := os.Open(tarPath)
 			require.NoError(t, err)
 			defer testhelper.MustClose(t, tar)
@@ -166,7 +168,7 @@ func TestBackupPartition_BackupExists(t *testing.T) {
 		testserver.WithBackupSink(backupSink),
 	)
 
-	_, err = ptnClient.BackupPartition(ctx, &gitalypb.BackupPartitionRequest{
+	resp, err := ptnClient.BackupPartition(ctx, &gitalypb.BackupPartitionRequest{
 		StorageName: "default",
 		PartitionId: "1",
 	})
@@ -180,11 +182,14 @@ func TestBackupPartition_BackupExists(t *testing.T) {
 		return
 	}
 
-	// Calling the same backup again should fail as it already exists
-	_, err = ptnClient.BackupPartition(ctx, &gitalypb.BackupPartitionRequest{
+	backupRelativePath := filepath.Join("partition-backups", "default", "1", "0000000000000.tar")
+	require.Equal(t, backupRelativePath, resp.GetBackupRelativePath())
+
+	// Calling the same backup again should not fail
+	resp, err = ptnClient.BackupPartition(ctx, &gitalypb.BackupPartitionRequest{
 		StorageName: "default",
 		PartitionId: "1",
 	})
-
-	testhelper.RequireGrpcError(t, structerr.NewAlreadyExists("there is an up-to-date backup for the given partition"), err)
+	require.NoError(t, err)
+	require.Equal(t, backupRelativePath, resp.GetBackupRelativePath())
 }
diff --git a/proto/go/gitalypb/partition.pb.go b/proto/go/gitalypb/partition.pb.go
index 7821d71f16..d6d54c7d15 100644
--- a/proto/go/gitalypb/partition.pb.go
+++ b/proto/go/gitalypb/partition.pb.go
@@ -83,6 +83,9 @@ type BackupPartitionResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
+
+	// backup_relative_path contains the relative path to the backup.
+	BackupRelativePath string `protobuf:"bytes,1,opt,name=backup_relative_path,json=backupRelativePath,proto3" json:"backup_relative_path,omitempty"`
 }
 
 func (x *BackupPartitionResponse) Reset() {
@@ -115,6 +118,13 @@ func (*BackupPartitionResponse) Descriptor() ([]byte, []int) {
 	return file_partition_proto_rawDescGZIP(), []int{1}
 }
 
+func (x *BackupPartitionResponse) GetBackupRelativePath() string {
+	if x != nil {
+		return x.BackupRelativePath
+	}
+	return ""
+}
+
 // ListPartitionsRequest is a request for the ListPartitions RPC.
 type ListPartitionsRequest struct {
 	state         protoimpl.MessageState
@@ -411,77 +421,80 @@ var file_partition_proto_rawDesc = []byte{
 	0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69,
 	0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0xa8,
 	0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64,
-	0x22, 0x19, 0x0a, 0x17, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74,
-	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8a, 0x01, 0x0a, 0x15,
-	0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
-	0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6, 0x2c,
-	0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x48,
-	0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x72,
-	0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x69, 0x74, 0x61,
-	0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72,
-	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73,
-	0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
-	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79,
-	0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74,
-	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x45, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67,
-	0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, 0xc6, 0x01,
-	0x0a, 0x24, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
-	0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67,
-	0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, 0xc6,
-	0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12,
-	0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09,
-	0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x61, 0x72,
-	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e,
-	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
-	0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74,
-	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa4, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38,
-	0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74,
-	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74,
-	0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
-	0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x62,
+	0x22, 0x4b, 0x0a, 0x17, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74,
+	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x62,
 	0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70,
-	0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x61, 0x63, 0x6b, 0x75,
-	0x70, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x22, 0x27, 0x0a,
-	0x25, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
-	0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd4, 0x02, 0x0a, 0x10, 0x50, 0x61, 0x72, 0x74, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x0f, 0x42,
-	0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e,
-	0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61,
-	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f,
-	0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61,
-	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x03, 0x12, 0x59, 0x0a, 0x0e, 0x4c, 0x69, 0x73,
-	0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x67, 0x69,
-	0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
-	0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x69, 0x74,
-	0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
-	0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04,
-	0x08, 0x02, 0x10, 0x02, 0x12, 0x86, 0x01, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50,
-	0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61,
-	0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e,
+	0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x61, 0x63, 0x6b, 0x75,
+	0x70, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x22, 0x8a, 0x01,
+	0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61,
+	0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88,
+	0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65,
+	0x12, 0x48, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x69,
+	0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x10, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x16, 0x4c,
+	0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
+	0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x69, 0x74, 0x61,
+	0x6c, 0x79, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x45, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x69,
+	0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x67,
+	0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x10, 0x70,
+	0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22,
+	0xc6, 0x01, 0x0a, 0x24, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74,
+	0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
+	0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72,
+	0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04,
+	0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d,
+	0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3b, 0x0a, 0x0a, 0x70,
+	0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x1b, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
+	0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x70, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa4, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72,
+	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x66, 0x6f,
+	0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52,
+	0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61,
+	0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x30, 0x0a,
+	0x14, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65,
+	0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x61, 0x63,
+	0x6b, 0x75, 0x70, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x22,
+	0x27, 0x0a, 0x25, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
+	0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd4, 0x02, 0x0a, 0x10, 0x50, 0x61, 0x72,
+	0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a,
+	0x0f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70,
+	0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x1f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70,
+	0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x03, 0x12, 0x59, 0x0a, 0x0e, 0x4c,
+	0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e,
+	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69,
+	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67,
+	0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74,
+	0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97,
+	0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x12, 0x86, 0x01, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74,
+	0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70,
+	0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c,
+	0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e,
 	0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42,
-	0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x43, 0x72,
-	0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63,
-	0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x42, 0x34, 0x5a,
-	0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c,
-	0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31,
-	0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c,
-	0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x42,
+	0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69,
+	0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f,
+	0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74,
+	0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/proto/partition.proto b/proto/partition.proto
index f17aa7cafc..561d7dd7dc 100644
--- a/proto/partition.proto
+++ b/proto/partition.proto
@@ -49,6 +49,8 @@ message BackupPartitionRequest {
 
 // BackupPartitionResponse is a response for the BackupPartition RPC.
 message BackupPartitionResponse {
+  // backup_relative_path contains the relative path to the backup.
+  string backup_relative_path = 1;
 }
 
 // ListPartitionsRequest is a request for the ListPartitions RPC.
-- 
GitLab


From f3b1b44c4c69ba2ef4e1f4f4a18bd585f99d59b0 Mon Sep 17 00:00:00 2001
From: Mustafa Bayar <mbayar@gitlab.com>
Date: Thu, 24 Oct 2024 11:51:59 +0300
Subject: [PATCH 4/4] backup: Call CreatePartitionBackupManifest after backup

After finishing the partition backup create, collect and send
the successful partitions to CreatePartitionBackupManifest RPC
to create a manifest file.
---
 internal/backup/partition_backup.go         | 48 +++++++++++++++------
 internal/backup/partition_backup_test.go    |  2 +-
 internal/cli/gitalybackup/partition_test.go |  2 +-
 3 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/internal/backup/partition_backup.go b/internal/backup/partition_backup.go
index f2db683d6e..ba2d7cb207 100644
--- a/internal/backup/partition_backup.go
+++ b/internal/backup/partition_backup.go
@@ -3,6 +3,7 @@ package backup
 import (
 	"context"
 	"fmt"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -11,8 +12,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v16/internal/log"
 	"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
 	"golang.org/x/sync/errgroup"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 const (
@@ -108,19 +108,31 @@ func (pbm *PartitionBackupManager) Create(ctx context.Context, serverInfo storag
 		return fmt.Errorf("create partition client: %w", err)
 	}
 
-	g, ctx := errgroup.WithContext(ctx)
+	g, gCtx := errgroup.WithContext(ctx)
 	partitionChan := make(chan *gitalypb.Partition)
 	var successCount, errCount atomic.Uint64
 
+	// Create a slice to store successful backups
+	var successfulBackups []*gitalypb.PartitionBackupInfo
+	var mu sync.Mutex
+
 	for i := 0; i < pbm.concurrencyLimit; i++ {
 		g.Go(func() error {
 			for partition := range partitionChan {
-				err := pbm.backupPartition(ctx, client, storageName, partition.GetId())
+				backupResponse, err := pbm.backupPartition(gCtx, client, storageName, partition.GetId())
 				if err != nil {
 					logger.Error(fmt.Sprintf("backup partition %s failed: %v", partition.GetId(), err))
 					errCount.Add(1)
 				} else {
 					successCount.Add(1)
+					// Store successful backup info
+					mu.Lock()
+					successfulBackups = append(successfulBackups, &gitalypb.PartitionBackupInfo{
+						PartitionId:        partition.GetId(),
+						BackupRelativePath: backupResponse.GetBackupRelativePath(),
+						Timestamp:          timestamppb.Now(),
+					})
+					mu.Unlock()
 				}
 			}
 			return nil
@@ -129,11 +141,26 @@ func (pbm *PartitionBackupManager) Create(ctx context.Context, serverInfo storag
 
 	g.Go(func() error {
 		defer close(partitionChan)
-		return pbm.listAndQueuePartitions(ctx, client, storageName, partitionChan, logger)
+		return pbm.listAndQueuePartitions(gCtx, client, storageName, partitionChan, logger)
 	})
 
 	err = g.Wait()
 
+	// Update manifest with successful backups
+	if len(successfulBackups) > 0 {
+		_, updateErr := client.CreatePartitionBackupManifest(ctx, &gitalypb.CreatePartitionBackupManifestRequest{
+			StorageName: storageName,
+			Timestamp:   timestamppb.Now(),
+			Partitions:  successfulBackups,
+		})
+		if updateErr != nil {
+			logger.Error(fmt.Sprintf("failed to update manifest: %v", updateErr))
+			// Decide whether to return this error or just log it
+		} else {
+			logger.Info("successfully updated backup manifest")
+		}
+	}
+
 	success := successCount.Load()
 	failure := errCount.Load()
 	logger.Info(fmt.Sprintf("Partition backup completed: %d succeeded, %d failed", success, failure))
@@ -145,21 +172,16 @@ func (pbm *PartitionBackupManager) Create(ctx context.Context, serverInfo storag
 	return err
 }
 
-func (pbm *PartitionBackupManager) backupPartition(ctx context.Context, client gitalypb.PartitionServiceClient, storageName string, partitionID string) error {
+func (pbm *PartitionBackupManager) backupPartition(ctx context.Context, client gitalypb.PartitionServiceClient, storageName string, partitionID string) (*gitalypb.BackupPartitionResponse, error) {
 	backupCtx, backupCancel := context.WithTimeout(ctx, pbm.backupTimeout)
 	defer backupCancel()
 
-	_, err := client.BackupPartition(backupCtx, &gitalypb.BackupPartitionRequest{
+	response, err := client.BackupPartition(backupCtx, &gitalypb.BackupPartitionRequest{
 		StorageName: storageName,
 		PartitionId: partitionID,
 	})
 
-	if status.Code(err) == codes.AlreadyExists {
-		// If there is an up to date backup for the partition, we consider this as successful operation.
-		return nil
-	}
-
-	return err
+	return response, err
 }
 
 func (pbm *PartitionBackupManager) listAndQueuePartitions(ctx context.Context, client gitalypb.PartitionServiceClient, storageName string, workQueue chan<- *gitalypb.Partition, logger log.Logger) error {
diff --git a/internal/backup/partition_backup_test.go b/internal/backup/partition_backup_test.go
index 2c4b37a552..0e1f184678 100644
--- a/internal/backup/partition_backup_test.go
+++ b/internal/backup/partition_backup_test.go
@@ -111,7 +111,7 @@ func TestPartitionBackup_CreateSuccess(t *testing.T) {
 			require.NoError(t, err)
 
 			for _, expectedArchive := range tc.expectedArchives {
-				tarPath := filepath.Join(backupRoot, cfg.Storages[0].Name, expectedArchive, storage.LSN(1).String()) + ".tar"
+				tarPath := filepath.Join(backupRoot, "partition-backups", cfg.Storages[0].Name, expectedArchive, storage.LSN(1).String()) + ".tar"
 				tar, err := os.Open(tarPath)
 				require.NoError(t, err)
 				testhelper.MustClose(t, tar)
diff --git a/internal/cli/gitalybackup/partition_test.go b/internal/cli/gitalybackup/partition_test.go
index 29e0bc7f24..7de474e8e2 100644
--- a/internal/cli/gitalybackup/partition_test.go
+++ b/internal/cli/gitalybackup/partition_test.go
@@ -110,7 +110,7 @@ func TestPartitionSubcommand_Create(t *testing.T) {
 			require.NoError(t, err)
 
 			lsn := storage.LSN(1)
-			tarPath := filepath.Join(path, cfg.Storages[0].Name, "2", lsn.String()) + ".tar"
+			tarPath := filepath.Join(path, "partition-backups", cfg.Storages[0].Name, "2", lsn.String()) + ".tar"
 			tar, err := os.Open(tarPath)
 			require.NoError(t, err)
 			defer testhelper.MustClose(t, tar)
-- 
GitLab