diff --git a/go.mod b/go.mod index c8374fa7a06a5f3487d2bcb02fa23c111a11f694..6a40d3b705638d54809ce600420673e0815d4913 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/urfave/cli/v3 v3.1.0 + github.com/vbauerster/mpb/v8 v8.9.3 gitlab.com/gitlab-org/labkit v1.23.2 go.etcd.io/raft/v3 v3.6.0 go.uber.org/automaxprocs v1.6.0 @@ -86,6 +87,8 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect @@ -172,7 +175,7 @@ require ( github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/montanaflynn/stats v0.7.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -187,7 +190,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/prometheus v0.54.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect diff --git a/go.sum b/go.sum index 5cb430d7499fc7bcd6d92801fec7445a4990dbe2..9a6b017d11698141c668c0f4309a05659fbc1dc1 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,10 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= @@ -504,8 +508,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= @@ -581,8 +585,9 @@ github.com/prometheus/prometheus v0.54.0 h1:6+VmEkohHcofl3W5LyRlhw1Lfm575w/aX6ZF github.com/prometheus/prometheus v0.54.0/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= @@ -642,6 +647,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v3 v3.1.0 h1:kQR+oiqpJkBAONxBjM4RWivD4AfPHL/f4vqe/gjYU8M= github.com/urfave/cli/v3 v3.1.0/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= +github.com/vbauerster/mpb/v8 v8.9.3 h1:PnMeF+sMvYv9u23l6DO6Q3+Mdj408mjLRXIzmUmU2Z8= +github.com/vbauerster/mpb/v8 v8.9.3/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= diff --git a/internal/cli/gitaly/subcmd_recovery.go b/internal/cli/gitaly/subcmd_recovery.go index 4642b60a2cc69fd6e4b89eeb8a8b208ba2ae3aee..ca3d72c7890b51fc11d3de795d1523da9d881087 100644 --- a/internal/cli/gitaly/subcmd_recovery.go +++ b/internal/cli/gitaly/subcmd_recovery.go @@ -16,6 +16,8 @@ import ( "time" "github.com/urfave/cli/v3" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" "gitlab.com/gitlab-org/gitaly/v16" "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" @@ -40,16 +42,18 @@ const ( flagAll = "all" flagParallel = "parallel" flagPartition = "partition" + flagVerbose = "verbose" ) type recoveryContext struct { - cmd *cli.Command - parallel int - nodeStorage storage.Storage - storageName string - partitions []storage.PartitionID - logEntryStore backup.LogEntryStore - cleanupFuncs *list.List + cmd *cli.Command + parallel int + nodeStorage storage.Storage + storageName string + partitions []storage.PartitionID + logEntryStore backup.LogEntryStore + cleanupFuncs *list.List + progressContainer *mpb.Progress } func newRecoveryCommand() *cli.Command { @@ -112,6 +116,10 @@ Example: gitaly recovery --config gitaly.config.toml status --storage default -- Name: flagRepository, Usage: "relative path to the repository", }, + &cli.BoolFlag{ + Name: flagVerbose, + Usage: "verbose output", + }, &cli.BoolFlag{ Name: flagAll, Usage: "runs the command for all partitions in the storage", @@ -201,32 +209,11 @@ func (rc *recoveryContext) printPartitionStatus(ctx context.Context, partitionID } } - entries := rc.logEntryStore.Query(backup.PartitionInfo{ - PartitionID: partitionID, - StorageName: rc.storageName, - }, appliedLSN+1) - - var lastLSN storage.LSN - discontinuity := false - for entries.Next(ctx) { - currentLSN := entries.LSN() - - // First iteration - if lastLSN == storage.LSN(0) { - lastLSN = currentLSN - continue - } - - if currentLSN != lastLSN+1 { - // We've found a gap - discontinuity = true - break - } - - lastLSN = currentLSN + lastLSN, discontinuity, err := rc.queryLogEntries(ctx, partitionID, appliedLSN) + if err != nil { + return fmt.Errorf("count archived log entries: %w", err) } - - if lastLSN == storage.LSN(0) { + if lastLSN == appliedLSN { buffer.WriteString("Available WAL backup entries: No entries found\n") } else { buffer.WriteString(fmt.Sprintf("Available WAL backup entries: up to LSN: %s\n", lastLSN.String())) @@ -237,10 +224,6 @@ func (rc *recoveryContext) printPartitionStatus(ctx context.Context, partitionID _, _ = buffer.WriteTo(rc.cmd.Writer) - if err := entries.Err(); err != nil { - return fmt.Errorf("query log entry store: %w", err) - } - return nil } @@ -269,10 +252,8 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro var successCount, errCount atomic.Uint64 for _, partitionID := range recoveryContext.partitions { g.Go(func() error { - fmt.Fprintf(cmd.Writer, "started processing partition %d\n", partitionID) err := recoveryContext.processPartition(ctx, tempDir, partitionID) if err != nil { - fmt.Fprintf(cmd.ErrWriter, "restore replay for partition %d failed: %v\n", partitionID, err) errCount.Add(1) } else { successCount.Add(1) @@ -282,10 +263,11 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro } err = g.Wait() + recoveryContext.progressContainer.Wait() success := successCount.Load() failure := errCount.Load() - fmt.Fprintf(recoveryContext.cmd.Writer, "recovery replay completed: %d succeeded, %d failed", success, failure) + fmt.Fprintf(recoveryContext.cmd.Writer, "recovery replay completed: %d succeeded, %d failed\n", success, failure) if err == nil && errCount.Load() > 0 { err = fmt.Errorf("recovery replay failed for %d out of %d partition(s)", failure, success+failure) @@ -294,7 +276,7 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro return err } -func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, partitionID storage.PartitionID) error { +func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, partitionID storage.PartitionID) (returnErr error) { var appliedLSN storage.LSN ptn, err := rc.nodeStorage.GetPartition(ctx, partitionID) @@ -311,6 +293,7 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, return fmt.Errorf("begin: %w", err) } appliedLSN = txn.SnapshotLSN() + relativePaths := txn.PartitionRelativePaths() err = txn.Rollback(ctx) if err != nil { return fmt.Errorf("rollback: %w", err) @@ -321,12 +304,68 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, StorageName: rc.storageName, } nextLSN := appliedLSN + 1 - finalLSN := appliedLSN + + lastLSN, _, err := rc.queryLogEntries(ctx, partitionID, appliedLSN) + if err != nil { + return fmt.Errorf("count archived log entries: %w", err) + } + logCount := lastLSN - appliedLSN + + verbose := rc.cmd.Bool(flagVerbose) + progressBarName := fmt.Sprintf("Partition %s", partitionID.String()) + if len(relativePaths) > 0 && verbose { + progressBarName = fmt.Sprintf("%s (%s)", progressBarName, relativePaths[0]) + } + doneMsg := "Done!" + if verbose { + if logCount == 0 { + doneMsg = fmt.Sprintf("Done! Already up to date at LSN %d", appliedLSN) + } else { + doneMsg = fmt.Sprintf("Done! Updated from LSN %d to %d", appliedLSN, lastLSN) + } + } + progressBar := rc.progressContainer.AddBar(int64(logCount), + mpb.BarFillerMiddleware(func(base mpb.BarFiller) mpb.BarFiller { + return mpb.BarFillerFunc(func(w io.Writer, st decor.Statistics) error { + if st.Completed || st.Aborted { + _, err := io.WriteString(w, doneMsg) + return err + } + return base.Fill(w, st) + }) + }), + mpb.PrependDecorators( + decor.Name(progressBarName, decor.WC{C: decor.DSyncSpaceR}), + decor.OnCompleteMetaOrOnAbortMeta( + decor.CountersNoUnit("%d / %d", decor.WCSyncWidth), + func(string) string { + return "" // remove progress counter when completed or aborted + }, + ), + ), + mpb.AppendDecorators( + decor.OnCompleteMetaOrOnAbortMeta( + decor.Percentage(), + func(string) string { + return "" // remove progress percentage when completed or aborted + }, + ), + ), + ) + progressBar.EnableTriggerComplete() + defer func() { + if returnErr != nil { + doneMsg = fmt.Sprintf("Failed! %v", returnErr) + progressBar.Abort(false) + } + + progressBar.Wait() + }() iterator := rc.logEntryStore.Query(partitionInfo, nextLSN) for iterator.Next(ctx) { if nextLSN != iterator.LSN() { - return fmt.Errorf("there is discontinuity in the WAL entries. Expected LSN: %s, Got: %s", nextLSN.String(), iterator.LSN().String()) + return fmt.Errorf("there is a discontinuity in the WAL entries at LSN: %s", nextLSN.String()) } reader, err := rc.logEntryStore.GetReader(ctx, partitionInfo, nextLSN) @@ -357,7 +396,7 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, ) } - finalLSN = nextLSN + progressBar.IncrBy(1) nextLSN++ } @@ -365,12 +404,6 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, return fmt.Errorf("query log entry store: %w", err) } - var buffer bytes.Buffer - buffer.WriteString("---------------------------------------------\n") - buffer.WriteString(fmt.Sprintf("Partition ID: %s - Applied LSN: %s\n", partitionID.String(), appliedLSN.String())) - buffer.WriteString(fmt.Sprintf("Successfully processed log entries up to LSN: %s\n", finalLSN.String())) - _, _ = buffer.WriteTo(rc.cmd.Writer) - return nil } @@ -485,13 +518,14 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon } }() - parallel := cmd.Int(flagParallel) - if parallel < 1 { - parallel = 1 - } + parallel := max(cmd.Int(flagParallel), 1) recoveryContext.parallel = int(parallel) - logger := log.ConfigureCommand() + // no-op logger to prevent distruptions to the progress container + logger, err := log.Configure(io.Discard, "text", "error") + if err != nil { + return recoveryContext, fmt.Errorf("configure logger: %w", err) + } cfg, err := loadConfig(cmd.String(flagConfig)) if err != nil { @@ -506,6 +540,9 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon return recoveryContext, fmt.Errorf("resolve sink: %w", err) } recoveryContext.logEntryStore = backup.NewLogEntryStore(sink) + recoveryContext.cleanupFuncs.PushFront(func() error { + return sink.Close() + }) runtimeDir, err := os.MkdirTemp("", "gitaly-recovery-*") if err != nil { @@ -530,7 +567,7 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon logger, ) if err != nil { - return recoveryContext, fmt.Errorf("new db manager: %w", err) + return recoveryContext, fmt.Errorf("new db manager: %w. Please stop Gitaly before running recovery commands", err) } recoveryContext.cleanupFuncs.PushFront(func() error { dbMgr.Close() @@ -642,6 +679,11 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon recoveryContext.partitions = append(recoveryContext.partitions, partitionID) } + recoveryContext.progressContainer = mpb.New( + mpb.WithOutput(cmd.Writer), + mpb.PopCompletedMode(), + ) + return recoveryContext, nil } @@ -656,6 +698,43 @@ func parsePartitionID(id *storage.PartitionID, value string) error { return nil } +func (rc *recoveryContext) queryLogEntries(ctx context.Context, partitionID storage.PartitionID, appliedLSN storage.LSN) (storage.LSN, bool, error) { + entries := rc.logEntryStore.Query(backup.PartitionInfo{ + PartitionID: partitionID, + StorageName: rc.storageName, + }, appliedLSN+1) + + var lastLSN storage.LSN + discontinuity := false + for entries.Next(ctx) { + currentLSN := entries.LSN() + + // First iteration + if lastLSN == storage.LSN(0) { + lastLSN = currentLSN + continue + } + + if currentLSN != lastLSN+1 { + // We've found a gap + discontinuity = true + break + } + + lastLSN = currentLSN + } + + if err := entries.Err(); err != nil { + return 0, false, fmt.Errorf("query log entry store: %w", err) + } + + if lastLSN == storage.LSN(0) { + lastLSN = appliedLSN + } + + return lastLSN, discontinuity, nil +} + func (rc *recoveryContext) Cleanup() error { var err error for i := rc.cleanupFuncs.Front(); i != nil; i = i.Next() { diff --git a/internal/cli/gitaly/subcmd_recovery_test.go b/internal/cli/gitaly/subcmd_recovery_test.go index 28d8422a133e2a387e42078ef7b6181d6750c361..93346c40f394a82ccc564c341dc5a8aa6f01cb44 100644 --- a/internal/cli/gitaly/subcmd_recovery_test.go +++ b/internal/cli/gitaly/subcmd_recovery_test.go @@ -420,15 +420,7 @@ func TestRecoveryCLI_replay(t *testing.T) { // It should return an error instead. // https://gitlab.com/gitlab-org/gitaly/-/issues/6478 expectedOutputs: []string{ - "started processing partition 42", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(42), - storage.LSN(0), - storage.LSN(0), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: nil, } @@ -444,15 +436,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-partition", storage.PartitionID(2).String()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(1), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 1}, } @@ -475,15 +459,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-partition", storage.PartitionID(2).String()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3}, } @@ -506,15 +482,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-repository", repo.GetRelativePath()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3}, } @@ -538,8 +506,6 @@ recovery replay completed: 1 succeeded, 0 failed`, args: []string{"-partition", storage.PartitionID(2).String()}, expectedErr: errors.New("exit status 1"), expectedOutputs: []string{ - "started processing partition 2", - "restore replay for partition 2 failed: there is discontinuity in the WAL entries. Expected LSN: 0000000000003, Got: 0000000000004", "recovery replay completed: 0 succeeded, 1 failed", "recovery replay failed for 1 out of 1 partition(s)", }, @@ -565,10 +531,8 @@ recovery replay completed: 1 succeeded, 0 failed`, args: []string{"-partition", storage.PartitionID(2).String()}, expectedErr: errors.New("exit status 1"), expectedOutputs: []string{ - "started processing partition 2", - "restore replay for partition 2 failed: failed to apply latest log entry: transaction processing stopped", - "ecovery replay completed: 0 succeeded, 1 failed", - `msg="partition failed" error="apply log entry: update: apply operations`, + "recovery replay completed: 0 succeeded, 1 failed", + "recovery replay failed for 1 out of 1 partition(s)", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 1}, } @@ -599,22 +563,6 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: opts.cfg.Storages[0].Name, args: []string{"-all", "-parallel", "2"}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), - "started processing partition 3", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s`, - storage.PartitionID(3), - storage.LSN(1), - storage.LSN(4), - ), "recovery replay completed: 2 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3, 3: 4},