From 40b162d825dbc7e1f8c88246b6af47580769e2d6 Mon Sep 17 00:00:00 2001 From: John Cai Date: Thu, 9 Nov 2023 11:26:11 -0500 Subject: [PATCH 1/2] proto: Add internal RunCommand RPC Add a RunCommand RPC that will be used with a new gitaly walker subcommand which will execute arbitrary commands on a set of repositories. --- .../service/internalgitaly/execute_command.go | 58 +++ proto/go/gitalypb/internal.pb.go | 353 ++++++++++++++++-- proto/go/gitalypb/internal_grpc.pb.go | 41 +- proto/internal.proto | 40 ++ 4 files changed, 457 insertions(+), 35 deletions(-) create mode 100644 internal/gitaly/service/internalgitaly/execute_command.go diff --git a/internal/gitaly/service/internalgitaly/execute_command.go b/internal/gitaly/service/internalgitaly/execute_command.go new file mode 100644 index 0000000000..278de303ec --- /dev/null +++ b/internal/gitaly/service/internalgitaly/execute_command.go @@ -0,0 +1,58 @@ +package internalgitaly + +import ( + "bytes" + "context" + + "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func (s *server) ExecuteCommand( + ctx context.Context, + req *gitalypb.ExecuteCommandRequest, +) (*gitalypb.ExecuteCommandResponse, error) { + repo := req.GetRepository() + + if repo == nil { + return nil, structerr.NewInvalidArgument("repository cannot be empty") + } + + repoPath, err := s.locator.GetRepoPath(repo) + if err != nil { + return nil, structerr.NewInternal("error getting repo path %w", err) + } + + var stdout, stderr bytes.Buffer + + cmd, err := command.New( + ctx, + s.logger, + req.GetArgs(), + command.WithStdout(&stdout), + command.WithStderr(&stderr), + command.WithDir(repoPath), + ) + if err != nil { + return nil, structerr.NewInternal("error creating command: %w", err) + } + + if err := cmd.Wait(); err != nil { + exitCode, found := command.ExitStatus(err) + if found { + return &gitalypb.ExecuteCommandResponse{ + ReturnCode: int32(exitCode), + Output: stdout.Bytes(), + ErrorOutput: stderr.Bytes(), + }, nil + } + + return nil, structerr.NewInternal("error running command: %w", err) + } + + return &gitalypb.ExecuteCommandResponse{ + Output: stdout.Bytes(), + ErrorOutput: stderr.Bytes(), + }, nil +} diff --git a/proto/go/gitalypb/internal.pb.go b/proto/go/gitalypb/internal.pb.go index 74b6573025..2e8ffa5c0c 100644 --- a/proto/go/gitalypb/internal.pb.go +++ b/proto/go/gitalypb/internal.pb.go @@ -130,6 +130,216 @@ func (x *WalkReposResponse) GetModificationTime() *timestamppb.Timestamp { return nil } +// GitCommand represents the command and arguments of a Git command +type GitCommand struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // name is the name of the Git command + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // action is the subcommand of the Git command + Action string `protobuf:"bytes,2,opt,name=action,proto3" json:"action,omitempty"` + // flags contains the flags passed to the Git command + Flags []string `protobuf:"bytes,3,rep,name=flags,proto3" json:"flags,omitempty"` + // args are the arguments passed to the Git command + Args []string `protobuf:"bytes,4,rep,name=args,proto3" json:"args,omitempty"` + // post_separator_args are the arguments after the post separator + PostSeparatorArgs []string `protobuf:"bytes,5,rep,name=post_separator_args,json=postSeparatorArgs,proto3" json:"post_separator_args,omitempty"` +} + +func (x *GitCommand) Reset() { + *x = GitCommand{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GitCommand) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GitCommand) ProtoMessage() {} + +func (x *GitCommand) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GitCommand.ProtoReflect.Descriptor instead. +func (*GitCommand) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +func (x *GitCommand) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *GitCommand) GetAction() string { + if x != nil { + return x.Action + } + return "" +} + +func (x *GitCommand) GetFlags() []string { + if x != nil { + return x.Flags + } + return nil +} + +func (x *GitCommand) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *GitCommand) GetPostSeparatorArgs() []string { + if x != nil { + return x.PostSeparatorArgs + } + return nil +} + +// RunCommandRequest is a request for the RunCommand RPC +type RunCommandRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // repository is the repository in which to run the command + Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"` + // git_command is the Git command to run on the repository + GitCommand *GitCommand `protobuf:"bytes,2,opt,name=git_command,json=gitCommand,proto3" json:"git_command,omitempty"` +} + +func (x *RunCommandRequest) Reset() { + *x = RunCommandRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RunCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunCommandRequest) ProtoMessage() {} + +func (x *RunCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunCommandRequest.ProtoReflect.Descriptor instead. +func (*RunCommandRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + +func (x *RunCommandRequest) GetRepository() *Repository { + if x != nil { + return x.Repository + } + return nil +} + +func (x *RunCommandRequest) GetGitCommand() *GitCommand { + if x != nil { + return x.GitCommand + } + return nil +} + +// RunCommandResponse is the resposne from the RunCommand RPC +type RunCommandResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // return_code is the return code of the command + ReturnCode int32 `protobuf:"varint,1,opt,name=return_code,json=returnCode,proto3" json:"return_code,omitempty"` + // output is the stdot from the command + Output []byte `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + // error_output is the stderr from the command + ErrorOutput []byte `protobuf:"bytes,3,opt,name=error_output,json=errorOutput,proto3" json:"error_output,omitempty"` +} + +func (x *RunCommandResponse) Reset() { + *x = RunCommandResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RunCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunCommandResponse) ProtoMessage() {} + +func (x *RunCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunCommandResponse.ProtoReflect.Descriptor instead. +func (*RunCommandResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{4} +} + +func (x *RunCommandResponse) GetReturnCode() int32 { + if x != nil { + return x.ReturnCode + } + return 0 +} + +func (x *RunCommandResponse) GetOutput() []byte { + if x != nil { + return x.Output + } + return nil +} + +func (x *RunCommandResponse) GetErrorOutput() []byte { + if x != nil { + return x.ErrorOutput + } + return nil +} + var File_internal_proto protoreflect.FileDescriptor var file_internal_proto_rawDesc = []byte{ @@ -137,29 +347,59 @@ var file_internal_proto_rawDesc = []byte{ 0x12, 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0a, 0x6c, 0x69, 0x6e, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x04, 0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, - 0x11, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x32, 0x5e, 0x0a, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x57, 0x61, 0x6c, 0x6b, - 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, - 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, - 0x08, 0x02, 0x10, 0x02, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, - 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0c, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, + 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x22, 0x81, 0x01, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, + 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, 0x11, 0x6d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x22, 0x92, 0x01, 0x0a, 0x0a, 0x47, 0x69, 0x74, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x70, 0x6f, 0x73, + 0x74, 0x5f, 0x73, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x61, 0x72, 0x67, 0x73, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x70, 0x6f, 0x73, 0x74, 0x53, 0x65, 0x70, 0x61, + 0x72, 0x61, 0x74, 0x6f, 0x72, 0x41, 0x72, 0x67, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x11, 0x52, 0x75, + 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x38, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x04, 0x98, 0xc6, 0x2c, 0x01, 0x52, 0x0a, 0x72, + 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x33, 0x0a, 0x0b, 0x67, 0x69, 0x74, + 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x47, 0x69, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x52, 0x0a, 0x67, 0x69, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x22, 0x70, + 0x0a, 0x12, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, + 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x32, 0xab, 0x01, 0x0a, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x69, 0x74, + 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, + 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x69, 0x74, + 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x30, + 0x01, 0x12, 0x4b, 0x0a, 0x0a, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, + 0x19, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x69, 0x74, + 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x42, 0x34, + 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, + 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, + 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, + 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -174,21 +414,29 @@ func file_internal_proto_rawDescGZIP() []byte { return file_internal_proto_rawDescData } -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_internal_proto_goTypes = []interface{}{ (*WalkReposRequest)(nil), // 0: gitaly.WalkReposRequest (*WalkReposResponse)(nil), // 1: gitaly.WalkReposResponse - (*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp + (*GitCommand)(nil), // 2: gitaly.GitCommand + (*RunCommandRequest)(nil), // 3: gitaly.RunCommandRequest + (*RunCommandResponse)(nil), // 4: gitaly.RunCommandResponse + (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*Repository)(nil), // 6: gitaly.Repository } var file_internal_proto_depIdxs = []int32{ - 2, // 0: gitaly.WalkReposResponse.modification_time:type_name -> google.protobuf.Timestamp - 0, // 1: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest - 1, // 2: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 5, // 0: gitaly.WalkReposResponse.modification_time:type_name -> google.protobuf.Timestamp + 6, // 1: gitaly.RunCommandRequest.repository:type_name -> gitaly.Repository + 2, // 2: gitaly.RunCommandRequest.git_command:type_name -> gitaly.GitCommand + 0, // 3: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest + 3, // 4: gitaly.InternalGitaly.RunCommand:input_type -> gitaly.RunCommandRequest + 1, // 5: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse + 4, // 6: gitaly.InternalGitaly.RunCommand:output_type -> gitaly.RunCommandResponse + 5, // [5:7] is the sub-list for method output_type + 3, // [3:5] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -197,6 +445,7 @@ func file_internal_proto_init() { return } file_lint_proto_init() + file_shared_proto_init() if !protoimpl.UnsafeEnabled { file_internal_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*WalkReposRequest); i { @@ -222,6 +471,42 @@ func file_internal_proto_init() { return nil } } + file_internal_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GitCommand); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RunCommandRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RunCommandResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -229,7 +514,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_internal_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/go/gitalypb/internal_grpc.pb.go b/proto/go/gitalypb/internal_grpc.pb.go index 817678ee46..bc3aa49e8e 100644 --- a/proto/go/gitalypb/internal_grpc.pb.go +++ b/proto/go/gitalypb/internal_grpc.pb.go @@ -25,6 +25,8 @@ type InternalGitalyClient interface { // WalkRepos walks the storage and streams back all known git repos on the // requested storage WalkRepos(ctx context.Context, in *WalkReposRequest, opts ...grpc.CallOption) (InternalGitaly_WalkReposClient, error) + // RunCommand runs an arbitrary Git command on a repository + RunCommand(ctx context.Context, in *RunCommandRequest, opts ...grpc.CallOption) (*RunCommandResponse, error) } type internalGitalyClient struct { @@ -67,6 +69,15 @@ func (x *internalGitalyWalkReposClient) Recv() (*WalkReposResponse, error) { return m, nil } +func (c *internalGitalyClient) RunCommand(ctx context.Context, in *RunCommandRequest, opts ...grpc.CallOption) (*RunCommandResponse, error) { + out := new(RunCommandResponse) + err := c.cc.Invoke(ctx, "/gitaly.InternalGitaly/RunCommand", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalGitalyServer is the server API for InternalGitaly service. // All implementations must embed UnimplementedInternalGitalyServer // for forward compatibility @@ -74,6 +85,8 @@ type InternalGitalyServer interface { // WalkRepos walks the storage and streams back all known git repos on the // requested storage WalkRepos(*WalkReposRequest, InternalGitaly_WalkReposServer) error + // RunCommand runs an arbitrary Git command on a repository + RunCommand(context.Context, *RunCommandRequest) (*RunCommandResponse, error) mustEmbedUnimplementedInternalGitalyServer() } @@ -84,6 +97,9 @@ type UnimplementedInternalGitalyServer struct { func (UnimplementedInternalGitalyServer) WalkRepos(*WalkReposRequest, InternalGitaly_WalkReposServer) error { return status.Errorf(codes.Unimplemented, "method WalkRepos not implemented") } +func (UnimplementedInternalGitalyServer) RunCommand(context.Context, *RunCommandRequest) (*RunCommandResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RunCommand not implemented") +} func (UnimplementedInternalGitalyServer) mustEmbedUnimplementedInternalGitalyServer() {} // UnsafeInternalGitalyServer may be embedded to opt out of forward compatibility for this service. @@ -118,13 +134,36 @@ func (x *internalGitalyWalkReposServer) Send(m *WalkReposResponse) error { return x.ServerStream.SendMsg(m) } +func _InternalGitaly_RunCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RunCommandRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalGitalyServer).RunCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/gitaly.InternalGitaly/RunCommand", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalGitalyServer).RunCommand(ctx, req.(*RunCommandRequest)) + } + return interceptor(ctx, in, info, handler) +} + // InternalGitaly_ServiceDesc is the grpc.ServiceDesc for InternalGitaly service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var InternalGitaly_ServiceDesc = grpc.ServiceDesc{ ServiceName: "gitaly.InternalGitaly", HandlerType: (*InternalGitalyServer)(nil), - Methods: []grpc.MethodDesc{}, + Methods: []grpc.MethodDesc{ + { + MethodName: "RunCommand", + Handler: _InternalGitaly_RunCommand_Handler, + }, + }, Streams: []grpc.StreamDesc{ { StreamName: "WalkRepos", diff --git a/proto/internal.proto b/proto/internal.proto index 47d8267779..c9b8aba3ff 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -4,6 +4,7 @@ package gitaly; import "google/protobuf/timestamp.proto"; import "lint.proto"; +import "shared.proto"; option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"; @@ -18,6 +19,13 @@ service InternalGitaly { scope_level: STORAGE }; } + + // RunCommand runs an arbitrary Git command on a repository + rpc RunCommand (RunCommandRequest) returns (RunCommandResponse) { + option (op_type) = { + op: ACCESSOR + }; + } } // WalkReposRequest ... @@ -35,3 +43,35 @@ message WalkReposResponse { // modified. google.protobuf.Timestamp modification_time = 2; } + +// GitCommand represents the command and arguments of a Git command +message GitCommand { + // name is the name of the Git command + string name = 1; + // action is the subcommand of the Git command + string action = 2; + // flags contains the flags passed to the Git command + repeated string flags = 3; + // args are the arguments passed to the Git command + repeated string args = 4; + // post_separator_args are the arguments after the post separator + repeated string post_separator_args = 5; +} + +// RunCommandRequest is a request for the RunCommand RPC +message RunCommandRequest { + // repository is the repository in which to run the command + Repository repository = 1 [(target_repository)=true]; + // git_command is the Git command to run on the repository + GitCommand git_command = 2; +} + +// RunCommandResponse is the resposne from the RunCommand RPC +message RunCommandResponse { + // return_code is the return code of the command + int32 return_code = 1; + // output is the stdot from the command + bytes output = 2; + // error_output is the stderr from the command + bytes error_output = 3; +} -- GitLab From 848723cc52d0eaec6d992a27bfe58754bc4e5847 Mon Sep 17 00:00:00 2001 From: John Cai Date: Mon, 13 Nov 2023 14:14:32 -0500 Subject: [PATCH 2/2] internalgitaly: Implement RunCommand RPC Implement the RunCommand RPC, which can be used to execute an arbitrary Git command on a given repository. --- .../service/internalgitaly/execute_command.go | 58 ---------- .../service/internalgitaly/run_command.go | 62 ++++++++++ .../internalgitaly/run_command_test.go | 109 ++++++++++++++++++ .../gitaly/service/internalgitaly/server.go | 15 ++- 4 files changed, 180 insertions(+), 64 deletions(-) delete mode 100644 internal/gitaly/service/internalgitaly/execute_command.go create mode 100644 internal/gitaly/service/internalgitaly/run_command.go create mode 100644 internal/gitaly/service/internalgitaly/run_command_test.go diff --git a/internal/gitaly/service/internalgitaly/execute_command.go b/internal/gitaly/service/internalgitaly/execute_command.go deleted file mode 100644 index 278de303ec..0000000000 --- a/internal/gitaly/service/internalgitaly/execute_command.go +++ /dev/null @@ -1,58 +0,0 @@ -package internalgitaly - -import ( - "bytes" - "context" - - "gitlab.com/gitlab-org/gitaly/v16/internal/command" - "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" - "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" -) - -func (s *server) ExecuteCommand( - ctx context.Context, - req *gitalypb.ExecuteCommandRequest, -) (*gitalypb.ExecuteCommandResponse, error) { - repo := req.GetRepository() - - if repo == nil { - return nil, structerr.NewInvalidArgument("repository cannot be empty") - } - - repoPath, err := s.locator.GetRepoPath(repo) - if err != nil { - return nil, structerr.NewInternal("error getting repo path %w", err) - } - - var stdout, stderr bytes.Buffer - - cmd, err := command.New( - ctx, - s.logger, - req.GetArgs(), - command.WithStdout(&stdout), - command.WithStderr(&stderr), - command.WithDir(repoPath), - ) - if err != nil { - return nil, structerr.NewInternal("error creating command: %w", err) - } - - if err := cmd.Wait(); err != nil { - exitCode, found := command.ExitStatus(err) - if found { - return &gitalypb.ExecuteCommandResponse{ - ReturnCode: int32(exitCode), - Output: stdout.Bytes(), - ErrorOutput: stderr.Bytes(), - }, nil - } - - return nil, structerr.NewInternal("error running command: %w", err) - } - - return &gitalypb.ExecuteCommandResponse{ - Output: stdout.Bytes(), - ErrorOutput: stderr.Bytes(), - }, nil -} diff --git a/internal/gitaly/service/internalgitaly/run_command.go b/internal/gitaly/service/internalgitaly/run_command.go new file mode 100644 index 0000000000..abea2c6ec7 --- /dev/null +++ b/internal/gitaly/service/internalgitaly/run_command.go @@ -0,0 +1,62 @@ +package internalgitaly + +import ( + "bytes" + "context" + + "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func (s *server) RunCommand( + ctx context.Context, + req *gitalypb.RunCommandRequest, +) (*gitalypb.RunCommandResponse, error) { + repo := req.GetRepository() + gitCmdParams := req.GetGitCommand() + + if repo == nil { + return nil, structerr.NewInvalidArgument("repository cannot be empty") + } + + var flags []git.Option + + for _, flag := range gitCmdParams.GetFlags() { + flags = append(flags, &git.Flag{Name: flag}) + } + + var stdout, stderr bytes.Buffer + + cmd, err := s.gitCmdFactory.New(ctx, repo, git.Command{ + Name: gitCmdParams.GetName(), + Action: gitCmdParams.GetAction(), + Flags: flags, + Args: gitCmdParams.GetArgs(), + PostSepArgs: gitCmdParams.GetPostSeparatorArgs(), + }, git.WithStdout(&stdout), git.WithStderr(&stderr), + ) + if err != nil { + return nil, structerr.NewInternal("error creating command: %w", err) + } + + if err := cmd.Wait(); err != nil { + exitCode, found := command.ExitStatus(err) + if found { + return &gitalypb.RunCommandResponse{ + ReturnCode: int32(exitCode), + Output: stdout.Bytes(), + ErrorOutput: stderr.Bytes(), + }, nil + } + + return nil, structerr.NewInternal("error running command: %w", err) + } + + return &gitalypb.RunCommandResponse{ + Output: []byte(text.ChompBytes(stdout.Bytes())), + ErrorOutput: []byte(text.ChompBytes(stderr.Bytes())), + }, nil +} diff --git a/internal/gitaly/service/internalgitaly/run_command_test.go b/internal/gitaly/service/internalgitaly/run_command_test.go new file mode 100644 index 0000000000..6e56ee7d43 --- /dev/null +++ b/internal/gitaly/service/internalgitaly/run_command_test.go @@ -0,0 +1,109 @@ +package internalgitaly + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func TestRunCommand(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + testRepo, testRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + RelativePath: "a", + }) + + testLocalRepo := localrepo.NewTestRepo(t, cfg, testRepo) + + commitID := gittest.WriteCommit( + t, + cfg, + testRepoPath, + gittest.WithMessage("hello world"), + gittest.WithBranch(git.DefaultBranch), + gittest.WithTreeEntries( + gittest.TreeEntry{ + Mode: "100644", + Path: ".gitattributes", + Content: "a/b/c foo=bar", + }, + ), + ) + commitData, err := testLocalRepo.ReadObject(ctx, commitID) + require.NoError(t, err) + + srv := NewServer(&service.Dependencies{ + Logger: testhelper.SharedLogger(t), + Cfg: cfg, + StorageLocator: config.NewLocator(cfg), + GitCmdFactory: gittest.NewCommandFactory(t, cfg), + }) + + client := setupInternalGitalyService(t, cfg, srv) + + testCases := []struct { + desc string + req *gitalypb.RunCommandRequest + expectedExitStatus int32 + expectedOutput []byte + }{ + { + desc: "git cat-file", + req: &gitalypb.RunCommandRequest{ + Repository: testRepo, + GitCommand: &gitalypb.GitCommand{ + Name: "cat-file", + Flags: []string{"-p"}, + Args: []string{git.DefaultBranch}, + }, + }, + expectedExitStatus: 0, + expectedOutput: commitData, + }, + { + desc: "reading a non-existent object", + req: &gitalypb.RunCommandRequest{ + Repository: testRepo, + GitCommand: &gitalypb.GitCommand{ + Name: "cat-file", + Flags: []string{"-p"}, + Args: []string{"does-not-exist"}, + }, + }, + expectedExitStatus: 128, + }, + { + desc: "attributes", + req: &gitalypb.RunCommandRequest{ + Repository: testRepo, + GitCommand: &gitalypb.GitCommand{ + Name: "check-attr", + Flags: []string{"--source=HEAD"}, + Args: []string{"foo"}, + PostSeparatorArgs: []string{"a/b/c"}, + }, + }, + expectedExitStatus: 0, + expectedOutput: []byte("a/b/c: foo: bar"), + }, + } + + for _, tc := range testCases { + resp, err := client.RunCommand(ctx, tc.req) + require.NoError(t, err) + require.Equal(t, tc.expectedExitStatus, resp.ReturnCode) + require.Equal(t, tc.expectedOutput, resp.Output) + } +} diff --git a/internal/gitaly/service/internalgitaly/server.go b/internal/gitaly/service/internalgitaly/server.go index a275ae252c..78cf9cb2e3 100644 --- a/internal/gitaly/service/internalgitaly/server.go +++ b/internal/gitaly/service/internalgitaly/server.go @@ -1,6 +1,7 @@ package internalgitaly import ( + "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" @@ -10,16 +11,18 @@ import ( type server struct { gitalypb.UnimplementedInternalGitalyServer - logger log.Logger - storages []config.Storage - locator storage.Locator + logger log.Logger + storages []config.Storage + locator storage.Locator + gitCmdFactory git.CommandFactory } // NewServer return an instance of the Gitaly service. func NewServer(deps *service.Dependencies) gitalypb.InternalGitalyServer { return &server{ - logger: deps.GetLogger(), - storages: deps.GetCfg().Storages, - locator: deps.GetLocator(), + logger: deps.GetLogger(), + storages: deps.GetCfg().Storages, + locator: deps.GetLocator(), + gitCmdFactory: deps.GetGitCmdFactory(), } } -- GitLab