From a290ffbf328a343bb2a54b49f48d93b47689adb8 Mon Sep 17 00:00:00 2001 From: Mustafa Bayar Date: Tue, 15 Apr 2025 14:40:54 +0200 Subject: [PATCH 1/2] subcmd_recovery: Refactor log entry store query Extract the log entry query to its own function which we will also use from replay command in the next commit --- internal/cli/gitaly/subcmd_recovery.go | 70 +++++++++++++++----------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/internal/cli/gitaly/subcmd_recovery.go b/internal/cli/gitaly/subcmd_recovery.go index 4642b60a2c..0a86d0c80d 100644 --- a/internal/cli/gitaly/subcmd_recovery.go +++ b/internal/cli/gitaly/subcmd_recovery.go @@ -201,32 +201,11 @@ func (rc *recoveryContext) printPartitionStatus(ctx context.Context, partitionID } } - entries := rc.logEntryStore.Query(backup.PartitionInfo{ - PartitionID: partitionID, - StorageName: rc.storageName, - }, appliedLSN+1) - - var lastLSN storage.LSN - discontinuity := false - for entries.Next(ctx) { - currentLSN := entries.LSN() - - // First iteration - if lastLSN == storage.LSN(0) { - lastLSN = currentLSN - continue - } - - if currentLSN != lastLSN+1 { - // We've found a gap - discontinuity = true - break - } - - lastLSN = currentLSN + lastLSN, discontinuity, err := rc.queryLogEntries(ctx, partitionID, appliedLSN) + if err != nil { + return fmt.Errorf("count archived log entries: %w", err) } - - if lastLSN == storage.LSN(0) { + if lastLSN == appliedLSN { buffer.WriteString("Available WAL backup entries: No entries found\n") } else { buffer.WriteString(fmt.Sprintf("Available WAL backup entries: up to LSN: %s\n", lastLSN.String())) @@ -237,10 +216,6 @@ func (rc *recoveryContext) printPartitionStatus(ctx context.Context, partitionID _, _ = buffer.WriteTo(rc.cmd.Writer) - if err := entries.Err(); err != nil { - return fmt.Errorf("query log entry store: %w", err) - } - return nil } @@ -656,6 +631,43 @@ func parsePartitionID(id *storage.PartitionID, value string) error { return nil } +func (rc *recoveryContext) queryLogEntries(ctx context.Context, partitionID storage.PartitionID, appliedLSN storage.LSN) (storage.LSN, bool, error) { + entries := rc.logEntryStore.Query(backup.PartitionInfo{ + PartitionID: partitionID, + StorageName: rc.storageName, + }, appliedLSN+1) + + var lastLSN storage.LSN + discontinuity := false + for entries.Next(ctx) { + currentLSN := entries.LSN() + + // First iteration + if lastLSN == storage.LSN(0) { + lastLSN = currentLSN + continue + } + + if currentLSN != lastLSN+1 { + // We've found a gap + discontinuity = true + break + } + + lastLSN = currentLSN + } + + if err := entries.Err(); err != nil { + return 0, false, fmt.Errorf("query log entry store: %w", err) + } + + if lastLSN == storage.LSN(0) { + lastLSN = appliedLSN + } + + return lastLSN, discontinuity, nil +} + func (rc *recoveryContext) Cleanup() error { var err error for i := rc.cleanupFuncs.Front(); i != nil; i = i.Next() { -- GitLab From 24451edc5361b6efe0bcacbe0752a7301cd4e54d Mon Sep 17 00:00:00 2001 From: Mustafa Bayar Date: Tue, 15 Apr 2025 15:22:18 +0200 Subject: [PATCH 2/2] subcmd_recovery: Add progress bar We have been logging outputs regarding the recovery process but it is not easy to track with regular logs since replay operation is expected to run in parallel and giving multiple updates per partition would make the output very confusing. Therefore we are switching to progress bar output format where update of each partition will be done in-place. Also added querying log store prior to initiating recovery. This will increase the requests made to backup sink, but this way we will be able to track to progress of per partition precisly. --- go.mod | 7 +- go.sum | 13 ++- internal/cli/gitaly/subcmd_recovery.go | 119 +++++++++++++++----- internal/cli/gitaly/subcmd_recovery_test.go | 64 +---------- 4 files changed, 114 insertions(+), 89 deletions(-) diff --git a/go.mod b/go.mod index c8374fa7a0..6a40d3b705 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/urfave/cli/v3 v3.1.0 + github.com/vbauerster/mpb/v8 v8.9.3 gitlab.com/gitlab-org/labkit v1.23.2 go.etcd.io/raft/v3 v3.6.0 go.uber.org/automaxprocs v1.6.0 @@ -86,6 +87,8 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect @@ -172,7 +175,7 @@ require ( github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/montanaflynn/stats v0.7.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -187,7 +190,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/prometheus v0.54.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect diff --git a/go.sum b/go.sum index 5cb430d749..9a6b017d11 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,10 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= @@ -504,8 +508,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= @@ -581,8 +585,9 @@ github.com/prometheus/prometheus v0.54.0 h1:6+VmEkohHcofl3W5LyRlhw1Lfm575w/aX6ZF github.com/prometheus/prometheus v0.54.0/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= @@ -642,6 +647,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v3 v3.1.0 h1:kQR+oiqpJkBAONxBjM4RWivD4AfPHL/f4vqe/gjYU8M= github.com/urfave/cli/v3 v3.1.0/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= +github.com/vbauerster/mpb/v8 v8.9.3 h1:PnMeF+sMvYv9u23l6DO6Q3+Mdj408mjLRXIzmUmU2Z8= +github.com/vbauerster/mpb/v8 v8.9.3/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= diff --git a/internal/cli/gitaly/subcmd_recovery.go b/internal/cli/gitaly/subcmd_recovery.go index 0a86d0c80d..ca3d72c789 100644 --- a/internal/cli/gitaly/subcmd_recovery.go +++ b/internal/cli/gitaly/subcmd_recovery.go @@ -16,6 +16,8 @@ import ( "time" "github.com/urfave/cli/v3" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" "gitlab.com/gitlab-org/gitaly/v16" "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" @@ -40,16 +42,18 @@ const ( flagAll = "all" flagParallel = "parallel" flagPartition = "partition" + flagVerbose = "verbose" ) type recoveryContext struct { - cmd *cli.Command - parallel int - nodeStorage storage.Storage - storageName string - partitions []storage.PartitionID - logEntryStore backup.LogEntryStore - cleanupFuncs *list.List + cmd *cli.Command + parallel int + nodeStorage storage.Storage + storageName string + partitions []storage.PartitionID + logEntryStore backup.LogEntryStore + cleanupFuncs *list.List + progressContainer *mpb.Progress } func newRecoveryCommand() *cli.Command { @@ -112,6 +116,10 @@ Example: gitaly recovery --config gitaly.config.toml status --storage default -- Name: flagRepository, Usage: "relative path to the repository", }, + &cli.BoolFlag{ + Name: flagVerbose, + Usage: "verbose output", + }, &cli.BoolFlag{ Name: flagAll, Usage: "runs the command for all partitions in the storage", @@ -244,10 +252,8 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro var successCount, errCount atomic.Uint64 for _, partitionID := range recoveryContext.partitions { g.Go(func() error { - fmt.Fprintf(cmd.Writer, "started processing partition %d\n", partitionID) err := recoveryContext.processPartition(ctx, tempDir, partitionID) if err != nil { - fmt.Fprintf(cmd.ErrWriter, "restore replay for partition %d failed: %v\n", partitionID, err) errCount.Add(1) } else { successCount.Add(1) @@ -257,10 +263,11 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro } err = g.Wait() + recoveryContext.progressContainer.Wait() success := successCount.Load() failure := errCount.Load() - fmt.Fprintf(recoveryContext.cmd.Writer, "recovery replay completed: %d succeeded, %d failed", success, failure) + fmt.Fprintf(recoveryContext.cmd.Writer, "recovery replay completed: %d succeeded, %d failed\n", success, failure) if err == nil && errCount.Load() > 0 { err = fmt.Errorf("recovery replay failed for %d out of %d partition(s)", failure, success+failure) @@ -269,7 +276,7 @@ func recoveryReplayAction(ctx context.Context, cmd *cli.Command) (returnErr erro return err } -func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, partitionID storage.PartitionID) error { +func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, partitionID storage.PartitionID) (returnErr error) { var appliedLSN storage.LSN ptn, err := rc.nodeStorage.GetPartition(ctx, partitionID) @@ -286,6 +293,7 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, return fmt.Errorf("begin: %w", err) } appliedLSN = txn.SnapshotLSN() + relativePaths := txn.PartitionRelativePaths() err = txn.Rollback(ctx) if err != nil { return fmt.Errorf("rollback: %w", err) @@ -296,12 +304,68 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, StorageName: rc.storageName, } nextLSN := appliedLSN + 1 - finalLSN := appliedLSN + + lastLSN, _, err := rc.queryLogEntries(ctx, partitionID, appliedLSN) + if err != nil { + return fmt.Errorf("count archived log entries: %w", err) + } + logCount := lastLSN - appliedLSN + + verbose := rc.cmd.Bool(flagVerbose) + progressBarName := fmt.Sprintf("Partition %s", partitionID.String()) + if len(relativePaths) > 0 && verbose { + progressBarName = fmt.Sprintf("%s (%s)", progressBarName, relativePaths[0]) + } + doneMsg := "Done!" + if verbose { + if logCount == 0 { + doneMsg = fmt.Sprintf("Done! Already up to date at LSN %d", appliedLSN) + } else { + doneMsg = fmt.Sprintf("Done! Updated from LSN %d to %d", appliedLSN, lastLSN) + } + } + progressBar := rc.progressContainer.AddBar(int64(logCount), + mpb.BarFillerMiddleware(func(base mpb.BarFiller) mpb.BarFiller { + return mpb.BarFillerFunc(func(w io.Writer, st decor.Statistics) error { + if st.Completed || st.Aborted { + _, err := io.WriteString(w, doneMsg) + return err + } + return base.Fill(w, st) + }) + }), + mpb.PrependDecorators( + decor.Name(progressBarName, decor.WC{C: decor.DSyncSpaceR}), + decor.OnCompleteMetaOrOnAbortMeta( + decor.CountersNoUnit("%d / %d", decor.WCSyncWidth), + func(string) string { + return "" // remove progress counter when completed or aborted + }, + ), + ), + mpb.AppendDecorators( + decor.OnCompleteMetaOrOnAbortMeta( + decor.Percentage(), + func(string) string { + return "" // remove progress percentage when completed or aborted + }, + ), + ), + ) + progressBar.EnableTriggerComplete() + defer func() { + if returnErr != nil { + doneMsg = fmt.Sprintf("Failed! %v", returnErr) + progressBar.Abort(false) + } + + progressBar.Wait() + }() iterator := rc.logEntryStore.Query(partitionInfo, nextLSN) for iterator.Next(ctx) { if nextLSN != iterator.LSN() { - return fmt.Errorf("there is discontinuity in the WAL entries. Expected LSN: %s, Got: %s", nextLSN.String(), iterator.LSN().String()) + return fmt.Errorf("there is a discontinuity in the WAL entries at LSN: %s", nextLSN.String()) } reader, err := rc.logEntryStore.GetReader(ctx, partitionInfo, nextLSN) @@ -332,7 +396,7 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, ) } - finalLSN = nextLSN + progressBar.IncrBy(1) nextLSN++ } @@ -340,12 +404,6 @@ func (rc *recoveryContext) processPartition(ctx context.Context, tempDir string, return fmt.Errorf("query log entry store: %w", err) } - var buffer bytes.Buffer - buffer.WriteString("---------------------------------------------\n") - buffer.WriteString(fmt.Sprintf("Partition ID: %s - Applied LSN: %s\n", partitionID.String(), appliedLSN.String())) - buffer.WriteString(fmt.Sprintf("Successfully processed log entries up to LSN: %s\n", finalLSN.String())) - _, _ = buffer.WriteTo(rc.cmd.Writer) - return nil } @@ -460,13 +518,14 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon } }() - parallel := cmd.Int(flagParallel) - if parallel < 1 { - parallel = 1 - } + parallel := max(cmd.Int(flagParallel), 1) recoveryContext.parallel = int(parallel) - logger := log.ConfigureCommand() + // no-op logger to prevent distruptions to the progress container + logger, err := log.Configure(io.Discard, "text", "error") + if err != nil { + return recoveryContext, fmt.Errorf("configure logger: %w", err) + } cfg, err := loadConfig(cmd.String(flagConfig)) if err != nil { @@ -481,6 +540,9 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon return recoveryContext, fmt.Errorf("resolve sink: %w", err) } recoveryContext.logEntryStore = backup.NewLogEntryStore(sink) + recoveryContext.cleanupFuncs.PushFront(func() error { + return sink.Close() + }) runtimeDir, err := os.MkdirTemp("", "gitaly-recovery-*") if err != nil { @@ -505,7 +567,7 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon logger, ) if err != nil { - return recoveryContext, fmt.Errorf("new db manager: %w", err) + return recoveryContext, fmt.Errorf("new db manager: %w. Please stop Gitaly before running recovery commands", err) } recoveryContext.cleanupFuncs.PushFront(func() error { dbMgr.Close() @@ -617,6 +679,11 @@ func setupRecoveryContext(ctx context.Context, cmd *cli.Command) (rc recoveryCon recoveryContext.partitions = append(recoveryContext.partitions, partitionID) } + recoveryContext.progressContainer = mpb.New( + mpb.WithOutput(cmd.Writer), + mpb.PopCompletedMode(), + ) + return recoveryContext, nil } diff --git a/internal/cli/gitaly/subcmd_recovery_test.go b/internal/cli/gitaly/subcmd_recovery_test.go index 28d8422a13..93346c40f3 100644 --- a/internal/cli/gitaly/subcmd_recovery_test.go +++ b/internal/cli/gitaly/subcmd_recovery_test.go @@ -420,15 +420,7 @@ func TestRecoveryCLI_replay(t *testing.T) { // It should return an error instead. // https://gitlab.com/gitlab-org/gitaly/-/issues/6478 expectedOutputs: []string{ - "started processing partition 42", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(42), - storage.LSN(0), - storage.LSN(0), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: nil, } @@ -444,15 +436,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-partition", storage.PartitionID(2).String()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(1), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 1}, } @@ -475,15 +459,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-partition", storage.PartitionID(2).String()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3}, } @@ -506,15 +482,7 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: repo.GetStorageName(), args: []string{"-repository", repo.GetRelativePath()}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s -recovery replay completed: 1 succeeded, 0 failed`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), + "recovery replay completed: 1 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3}, } @@ -538,8 +506,6 @@ recovery replay completed: 1 succeeded, 0 failed`, args: []string{"-partition", storage.PartitionID(2).String()}, expectedErr: errors.New("exit status 1"), expectedOutputs: []string{ - "started processing partition 2", - "restore replay for partition 2 failed: there is discontinuity in the WAL entries. Expected LSN: 0000000000003, Got: 0000000000004", "recovery replay completed: 0 succeeded, 1 failed", "recovery replay failed for 1 out of 1 partition(s)", }, @@ -565,10 +531,8 @@ recovery replay completed: 1 succeeded, 0 failed`, args: []string{"-partition", storage.PartitionID(2).String()}, expectedErr: errors.New("exit status 1"), expectedOutputs: []string{ - "started processing partition 2", - "restore replay for partition 2 failed: failed to apply latest log entry: transaction processing stopped", - "ecovery replay completed: 0 succeeded, 1 failed", - `msg="partition failed" error="apply log entry: update: apply operations`, + "recovery replay completed: 0 succeeded, 1 failed", + "recovery replay failed for 1 out of 1 partition(s)", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 1}, } @@ -599,22 +563,6 @@ recovery replay completed: 1 succeeded, 0 failed`, storageName: opts.cfg.Storages[0].Name, args: []string{"-all", "-parallel", "2"}, expectedOutputs: []string{ - "started processing partition 2", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s`, - storage.PartitionID(2), - storage.LSN(1), - storage.LSN(3), - ), - "started processing partition 3", - fmt.Sprintf(`--------------------------------------------- -Partition ID: %s - Applied LSN: %s -Successfully processed log entries up to LSN: %s`, - storage.PartitionID(3), - storage.LSN(1), - storage.LSN(4), - ), "recovery replay completed: 2 succeeded, 0 failed", }, expectedLSN: map[storage.PartitionID]storage.LSN{2: 3, 3: 4}, -- GitLab