diff --git a/cmd/gitaly-git2go/resolve_conflicts.go b/cmd/gitaly-git2go/resolve_conflicts.go new file mode 100644 index 0000000000000000000000000000000000000000..ba1d516cbc40cb343eb45bc20f12c20992de218b --- /dev/null +++ b/cmd/gitaly-git2go/resolve_conflicts.go @@ -0,0 +1,221 @@ +// +build static,system_libgit2 + +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + git "github.com/libgit2/git2go/v30" + "gitlab.com/gitlab-org/gitaly/internal/git/conflict" + "gitlab.com/gitlab-org/gitaly/internal/git2go" +) + +type resolveSubcommand struct { + request string +} + +func (cmd *resolveSubcommand) Flags() *flag.FlagSet { + flags := flag.NewFlagSet("resolve", flag.ExitOnError) + flags.StringVar(&cmd.request, "request", "", "git2go.MergeCommand") + return flags +} + +func (cmd resolveSubcommand) Run() error { + request, err := git2go.ResolveCommandFromSerialized(cmd.request) + if err != nil { + return err + } + + if request.AuthorDate.IsZero() { + request.AuthorDate = time.Now() + } + + repo, err := git.OpenRepository(request.Repository) + if err != nil { + return fmt.Errorf("could not open repository: %w", err) + } + + ours, err := lookupCommit(repo, request.Ours) + if err != nil { + return fmt.Errorf("could not lookup commit %q: %w", request.Ours, err) + } + + theirs, err := lookupCommit(repo, request.Theirs) + if err != nil { + return fmt.Errorf("could not lookup commit %q: %w", request.Theirs, err) + } + + index, err := repo.MergeCommits(ours, theirs, nil) + if err != nil { + return fmt.Errorf("could not merge commits: %w", err) + } + + ci, err := index.ConflictIterator() + if err != nil { + return err + } + + type paths struct { + theirs, ours string + } + conflicts := map[paths]git.IndexConflict{} + + for c, err := ci.Next(); err != nil; c, err = ci.Next() { + if c.Our.Path == "" || c.Their.Path == "" { + return errors.New("conflict side missing") + } + + k := paths{ + theirs: c.Their.Path, + ours: c.Our.Path, + } + conflicts[k] = c + } + + for _, r := range request.Resolutions { + c, ok := conflicts[paths{ + theirs: r.OldPath, + ours: r.NewPath, + }] + if !ok { + continue + } + + odb, err := repo.Odb() + if err != nil { + return err + } + + mfr, err := mergeFileResult(odb, c) + if err != nil { + return err + } + + f, err := conflict.Parse( + bytes.NewReader(mfr.Contents), + c.Our.Path, + c.Their.Path, + c.Ancestor.Path, + ) + if err != nil { + return err + } + + resolvedBlob, err := f.Resolve(r) + if err != nil { + return err + } + + resolvedBlobOID, err := odb.Write(resolvedBlob, git.ObjectBlob) + if err != nil { + return err + } + + ourResolvedEntry := *c.Our // copy by value + ourResolvedEntry.Id = resolvedBlobOID + if err := index.Add(&ourResolvedEntry); err != nil { + return err + } + + // TODO: verify we need to remove conflict + if err := index.RemoveConflict(ourResolvedEntry.Path); err != nil { + return err + } + } + + if index.HasConflicts() { + ci, err := index.ConflictIterator() + if err != nil { + return err + } + + var conflictPaths []string + for { + c, err := ci.Next() + if err != nil { + return err + } + conflictPaths = append(conflictPaths, c.Ancestor.Path) + } + + return fmt.Errorf( + "Missing resolutions for the following files: %s", + strings.Join(conflictPaths, ", "), + ) + } + + tree, err := index.WriteTreeTo(repo) + if err != nil { + return err + } + + committer := git.Signature{ + Name: sanitizeSignatureInfo(request.AuthorName), + Email: sanitizeSignatureInfo(request.AuthorMail), + When: request.AuthorDate, + } + + commit, err := repo.CreateCommitFromIds("", &committer, &committer, request.Message, tree, ours.Id(), theirs.Id()) + if err != nil { + return fmt.Errorf("could not create resolve conflict commit: %w", err) + } + + response := git2go.ResolveResult{ + git2go.MergeResult{ + CommitID: commit.String(), + }, + } + + if err := response.SerializeTo(os.Stdout); err != nil { + return err + } + + return nil +} + +func mergeFileResult(odb *git.Odb, c git.IndexConflict) (*git.MergeFileResult, error) { + ancestorBlob, err := odb.Read(c.Ancestor.Id) + if err != nil { + return nil, err + } + + ourBlob, err := odb.Read(c.Our.Id) + if err != nil { + return nil, err + } + + theirBlob, err := odb.Read(c.Their.Id) + if err != nil { + return nil, err + } + + mfr, err := git.MergeFile( + git.MergeFileInput{ + Path: c.Ancestor.Path, + Mode: uint(c.Ancestor.Mode), + Contents: ancestorBlob.Data(), + }, + git.MergeFileInput{ + Path: c.Our.Path, + Mode: uint(c.Our.Mode), + Contents: ourBlob.Data(), + }, + git.MergeFileInput{ + Path: c.Their.Path, + Mode: uint(c.Their.Mode), + Contents: theirBlob.Data(), + }, + nil, + ) + if err != nil { + return nil, err + } + + return mfr, nil +} diff --git a/go.mod b/go.mod index a862936d6187566048aa2cd91d1603bd865e9fa3..a0bda0dfb7ac3d116d8f5e7d32d1589b0968435a 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( golang.org/x/sys v0.0.0-20200918174421-af09f7315aff golang.org/x/text v0.3.3 // indirect google.golang.org/grpc v1.24.0 + gopkg.in/errgo.v2 v2.1.0 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index d1071f46285fd0f5a4a3c7ff2d3005f413f0aa9e..25b882e1b16cf07c6704d9ec4308d677412c2247 100644 --- a/go.sum +++ b/go.sum @@ -605,6 +605,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/internal/git/conflict/parser.go b/internal/git/conflict/parser.go new file mode 100644 index 0000000000000000000000000000000000000000..d0938c7d6c42cee9db5bc1b3fd9086201e4e420a --- /dev/null +++ b/internal/git/conflict/parser.go @@ -0,0 +1,200 @@ +package conflict + +// This file is a direct port of Ruby source previously hosted at +// ruby/lib/gitlab/git/conflict/parser.rb (git show fb5717dd5567082) +// ruby/lib/gitlab/git/conflict/file.rb (git show 6f787458251b5f) + +import ( + "bufio" + "bytes" + "crypto/sha1" + "errors" + "fmt" + "io" +) + +// Errors that can occur during parsing of a merge conflict file +var ( + ErrUnmergeableFile = errors.New("merging is not supported for file") + ErrUnexpectedDelimiter = errors.New("unexpected conflict delimiter") + ErrMissingEndDelimiter = errors.New("missing last delimiter") +) + +type section uint + +const ( + sectionNone = section(iota) + sectionOld + sectionNew + sectionNoNewline +) + +const fileLimit = 200 * (1 << 10) // 200k + +type line struct { + objIndex uint // where are we in the object? + oldIndex uint // where are we in the old file? + newIndex uint // where are we in the new file? + + payload string // actual line contents (minus the newline) + section section +} + +// File contains an ordered list of lines with metadata about potential +// conflicts. +type File struct { + path string + lines []line +} + +func (f File) sectionID(l line) string { + pathSHA1 := sha1.Sum([]byte(f.path)) + return fmt.Sprintf("%x_%d_%d", pathSHA1, l.oldIndex, l.newIndex) +} + +// Resolution indicates how to resolve a conflict +type Resolution struct { + OldPath string `json:"old_path"` + NewPath string `json:"new_path"` + + // key is a sectionID, value is "head" or "origin" + Sections map[string]string `json:"sections"` +} + +const ( + head = "head" + origin = "origin" +) + +// Resolve will iterate through each conflict line and replace it with the +// specified resolution +func (f File) Resolve(resolution Resolution) ([]byte, error) { + var sectionID string + b := bytes.NewBuffer(nil) + + for _, l := range f.lines { + if l.section == sectionNone { + sectionID = "" + if _, err := b.WriteString(l.payload + "\n"); err != nil { + return nil, err + } + continue + } + + if sectionID == "" { + sectionID = f.sectionID(l) + } + + r, ok := resolution.Sections[sectionID] + if !ok { + return nil, fmt.Errorf("missing resolution for section ID: %s", sectionID) + } + + switch r { + case head: + if l.section != sectionNew { + continue + } + case origin: + if l.section != sectionOld { + continue + } + default: + return nil, fmt.Errorf("unknown resolution: %q", r) + } + + if _, err := b.WriteString(l.payload); err != nil { + return nil, err + } + + if l.section == sectionNoNewline { + continue + } + if _, err := b.WriteString("\n"); err != nil { + return nil, err + } + } + + return b.Bytes(), nil +} + +// Parse will read each line and maintain which conflict section it belongs to +func Parse(src io.Reader, ourPath, theirPath, parentPath string) (File, error) { + var ( + // conflict markers + start = "<<<<<<< " + ourPath + middle = "=======" + end = ">>>>>>> " + theirPath + noNewline = `\` + + f = File{path: parentPath} + objIndex, oldIndex, newIndex uint = 0, 1, 1 + currentSection section + bytesRead int + + s = bufio.NewScanner(src) + ) + + s.Buffer(make([]byte, 4096), fileLimit) // allow for line scanning up to the file limit + + s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { + bytesRead += len(data) + if bytesRead >= fileLimit { + return 0, nil, ErrUnmergeableFile + } + return bufio.ScanLines(data, atEOF) + }) + + for s.Scan() { + switch l := s.Text(); l { + case start: + if currentSection != sectionNone { + return File{}, ErrUnexpectedDelimiter + } + currentSection = sectionNew + case middle: + if currentSection != sectionNew { + return File{}, ErrUnexpectedDelimiter + } + currentSection = sectionOld + case end: + if currentSection != sectionOld { + return File{}, ErrUnexpectedDelimiter + } + currentSection = sectionNone + case noNewline: + currentSection = sectionNoNewline + fallthrough + default: + f.lines = append(f.lines, line{ + objIndex: objIndex, + oldIndex: oldIndex, + newIndex: newIndex, + payload: l, + section: currentSection, + }) + + objIndex++ + if currentSection != sectionNew { + oldIndex++ + } + if currentSection != sectionOld { + newIndex++ + } + } + } + + if err := s.Err(); err != nil { + return File{}, err + } + + if currentSection == sectionOld || currentSection == sectionNew { + return File{}, ErrMissingEndDelimiter + } + + if bytesRead == 0 { + return File{}, ErrUnmergeableFile // typically a binary file + } + + return f, nil +} diff --git a/internal/git/conflict/parser_test.go b/internal/git/conflict/parser_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b15cc84cc37d36591150ba10c344db4724ae5c48 --- /dev/null +++ b/internal/git/conflict/parser_test.go @@ -0,0 +1,127 @@ +package conflict + +import ( + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFile_Resolve(t *testing.T) { + for _, tt := range []struct { + name string + ourPath, theirPath, parentPath string + conflictFile io.Reader + parseErr error + resolution Resolution + resolveErr error + expect string + }{ + { + name: "ours", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(`# this file is very conflicted +<<<<<<< conflict.txt +we want this line +======= +but they want this line +>>>>>>> conflict.txt +we can both agree on this line though +`), + resolution: Resolution{ + NewPath: "conflict.txt", + OldPath: "conflict.txt", + Sections: map[string]string{ + "dc1c302824bab8da29f7c06fec1c77cf16b975e6_2_2": "head", + }, + }, + expect: `# this file is very conflicted +we want this line +we can both agree on this line though +`, + }, + { + name: "theirs", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(`# this file is very conflicted +<<<<<<< conflict.txt +we want this line +======= +but they want this line +>>>>>>> conflict.txt +we can both agree on this line though +`), + resolution: Resolution{ + NewPath: "conflict.txt", + OldPath: "conflict.txt", + Sections: map[string]string{ + "dc1c302824bab8da29f7c06fec1c77cf16b975e6_2_2": "origin", + }, + }, + expect: `# this file is very conflicted +but they want this line +we can both agree on this line though +`, + }, + { + name: "UnexpectedDelimiter", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(`# this file is very conflicted +<<<<<<< conflict.txt +we want this line +<<<<<<< conflict.txt +======= +but they want this line +>>>>>>> conflict.txt +we can both agree on this line though +`), + parseErr: ErrUnexpectedDelimiter, + }, + { + name: "ErrMissingEndDelimiter", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(`# this file is very conflicted +<<<<<<< conflict.txt +we want this line +======= +but they want this line +we can both agree on this line though +`), + parseErr: ErrMissingEndDelimiter, + }, + { + name: "ErrUnmergeableFile over file limit", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(strings.Repeat("x", fileLimit+1)), + parseErr: ErrUnmergeableFile, + }, + { + name: "ErrUnmergeableFile empty file", + ourPath: "conflict.txt", + theirPath: "conflict.txt", + parentPath: "conflict.txt", + conflictFile: strings.NewReader(""), + parseErr: ErrUnmergeableFile, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f, err := Parse(tt.conflictFile, tt.ourPath, tt.ourPath, tt.ourPath) + require.Equal(t, tt.parseErr, err) + + actual, err := f.Resolve(tt.resolution) + require.Equal(t, tt.resolveErr, err) + require.Equal(t, tt.expect, string(actual)) + }) + } +} diff --git a/internal/git2go/resolve_conflicts.go b/internal/git2go/resolve_conflicts.go new file mode 100644 index 0000000000000000000000000000000000000000..bc1057526224bc9686228badcaf853baa131e872 --- /dev/null +++ b/internal/git2go/resolve_conflicts.go @@ -0,0 +1,54 @@ +package git2go + +import ( + "context" + "fmt" + + "gitlab.com/gitlab-org/gitaly/internal/git/conflict" + "gitlab.com/gitlab-org/gitaly/internal/gitaly/config" +) + +type ResolveCommand struct { + MergeCommand `json:"merge_command"` + Resolutions []conflict.Resolution `json:"conflict_files"` +} + +type ResolveResult struct { + MergeResult `json:"merge_result"` +} + +func ResolveCommandFromSerialized(serialized string) (ResolveCommand, error) { + var request ResolveCommand + if err := deserialize(serialized, &request); err != nil { + return ResolveCommand{}, err + } + + if err := request.verify(); err != nil { + return ResolveCommand{}, fmt.Errorf("resolve: %w: %s", ErrInvalidArgument, err.Error()) + } + + return request, nil +} + +func (r ResolveCommand) Run(ctx context.Context, cfg config.Cfg) (ResolveResult, error) { + if err := r.verify(); err != nil { + return ResolveResult{}, fmt.Errorf("resolve: %w: %s", ErrInvalidArgument, err.Error()) + } + + serialized, err := serialize(r) + if err != nil { + return ResolveResult{}, err + } + + stdout, err := run(ctx, cfg, "resolve", serialized) + if err != nil { + return ResolveResult{}, err + } + + var response ResolveResult + if err := deserialize(stdout, &response); err != nil { + return ResolveResult{}, err + } + + return response, nil +} diff --git a/internal/gitaly/service/conflicts/resolve_conflicts.go b/internal/gitaly/service/conflicts/resolve_conflicts.go index d8734ef8d525e6a557a1dac7e283f237bf43f2f6..5f6aa3eee59eedbe41fbdcf1bb0101df27793727 100644 --- a/internal/gitaly/service/conflicts/resolve_conflicts.go +++ b/internal/gitaly/service/conflicts/resolve_conflicts.go @@ -1,12 +1,25 @@ package conflicts import ( + "bytes" + "context" + "encoding/json" "fmt" + "io" + "sort" + "gitlab.com/gitlab-org/gitaly/client" + "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/conflict" + "gitlab.com/gitlab-org/gitaly/internal/git2go" "gitlab.com/gitlab-org/gitaly/internal/gitaly/rubyserver" + "gitlab.com/gitlab-org/gitaly/internal/gitalyssh" + "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "gopkg.in/errgo.v2/errors" ) func (s *server) ResolveConflicts(stream gitalypb.ConflictsService_ResolveConflictsServer) error { @@ -24,6 +37,10 @@ func (s *server) ResolveConflicts(stream gitalypb.ConflictsService_ResolveConfli return status.Errorf(codes.InvalidArgument, "ResolveConflicts: %v", err) } + if featureflag.IsEnabled(stream.Context(), featureflag.GoResolveConflicts) { + return s.resolveConflicts(header, stream) + } + ctx := stream.Context() client, err := s.ruby.ConflictsServiceClient(ctx) if err != nil { @@ -89,3 +106,153 @@ func validateResolveConflictsHeader(header *gitalypb.ResolveConflictsRequestHead return nil } + +func (s *server) resolveConflicts(header *gitalypb.ResolveConflictsRequestHeader, stream gitalypb.ConflictsService_ResolveConflictsServer) error { + b := bytes.NewBuffer(nil) + for { + req, err := stream.Recv() + switch err { + case io.EOF: + return nil + case nil: + // do nothing, continue + default: + return err + } + + if _, err := b.Write(req.GetFilesJson()); err != nil { + return err + } + } + + var resolutions []conflict.Resolution + if err := json.NewDecoder(b).Decode(&resolutions); err != nil { + return err + } + + if err := s.repoWithBranchCommit( + stream.Context(), + header.GetRepository(), + header.GetTargetRepository(), + header.SourceBranch, + header.TargetBranch, + ); err != nil { + return err + } + + repoPath, err := s.locator.GetRepoPath(header.GetRepository()) + if err != nil { + return err + } + + _, err = git2go.ResolveCommand{ + MergeCommand: git2go.MergeCommand{ + Repository: repoPath, + AuthorName: string(header.User.Name), + AuthorMail: string(header.User.Email), + Message: string(header.CommitMessage), + Ours: header.GetOurCommitOid(), + Theirs: header.GetTheirCommitOid(), + }, + Resolutions: resolutions, + }.Run(stream.Context(), s.cfg) + if err != nil { + if errors.Is(git2go.ErrInvalidArgument)(err) { + return helper.ErrInvalidArgument(err) + } + return err + } + + return stream.SendAndClose(&gitalypb.ResolveConflictsResponse{ + ResolutionError: err.Error(), + }) +} + +func sameRepo(left, right *gitalypb.Repository) bool { + lgaod := left.GetGitAlternateObjectDirectories() + rgaod := right.GetGitAlternateObjectDirectories() + if len(lgaod) != len(rgaod) { + return false + } + sort.Strings(lgaod) + sort.Strings(rgaod) + for i := 0; i < len(lgaod); i++ { + if lgaod[i] != rgaod[i] { + return false + } + } + if left.GetGitObjectDirectory() != right.GetGitObjectDirectory() { + return false + } + if left.GetRelativePath() != right.GetRelativePath() { + return false + } + if left.GetStorageName() != right.GetStorageName() { + return false + } + return true +} + +const gitalyInternalURL = "ssh://gitaly/internal.git" + +// repoWithCommit ensures that the source repo contains the same commit we +// hope to merge with from the target branch, else it will be fetched from the +// target repo. This is necessary since all merge/resolve logic occurs on the +// same filesystem +func (s *server) repoWithBranchCommit(ctx context.Context, srcRepo, targetRepo *gitalypb.Repository, srcBranch, targetBranch []byte) error { + src := git.NewRepository(srcRepo) + if sameRepo(srcRepo, targetRepo) { + _, err := src.ResolveRefish(ctx, string(targetBranch)) + return err + } + + target, err := git.NewRemoteRepository(ctx, targetRepo, client.NewPool()) + if err != nil { + return err + } + + oid, err := target.ResolveRefish(ctx, string(targetBranch)) + if err != nil { + return err + } + + ok, err := src.ContainsRef(ctx, oid) + if err != nil { + return err + } + if ok { + // target branch commit already exists in source repo; nothing + // to do + return err + } + + env, err := gitalyssh.UploadPackEnv(ctx, &gitalypb.SSHUploadPackRequest{Repository: targetRepo}) + if err != nil { + return err + } + // to enable fetching a specific SHA: + env = append(env, "uploadpack.allowAnySHA1InWant=true") + + srcRepoPath, err := s.locator.GetRepoPath(srcRepo) + if err != nil { + return err + } + + cmd, err := git.SafeBareCmd(ctx, git.CmdStream{}, env, + []git.Option{git.ValueFlag{"--git-dir", srcRepoPath}}, + git.SubCmd{ + Name: "fetch", + Flags: []git.Option{git.Flag{Name: "--no-tags"}}, + Args: []string{gitalyInternalURL, oid}, + }, + ) + if err != nil { + return err + } + + if err := cmd.Wait(); err != nil { + return err + } + + return nil +} diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go index 9f1af69e4d767e056e2d793672c0e885f1e89154..aa6d8af452cf8a3be2a75f24821349efaf0c8218 100644 --- a/internal/metadata/featureflag/feature_flags.go +++ b/internal/metadata/featureflag/feature_flags.go @@ -38,6 +38,8 @@ var ( GoUserSquash = FeatureFlag{Name: "go_user_squash", OnByDefault: false} // GoListConflictFiles enables the Go implementation of ListConflictFiles GoListConflictFiles = FeatureFlag{Name: "go_list_conflict_files", OnByDefault: false} + // GoResolveConflicts enables the Go implementation of ResolveConflicts + GoResolveConflicts = FeatureFlag{Name: "go_resolve_conflicts", OnByDefault: false} ) // All includes all feature flags.