dailz/go-kv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: stop WAL recovery from advancing MANIFEST or using CURRENT (C2+C3)

Browse Source 
Phase 1 default state had two data-loss paths in WAL recovery.

C2: resolveRecoverySegmentID fell back to CURRENT when MANIFEST=0.
    Since segment_manager writes CURRENT on every segment create/rotate,
    the first recovery in Phase 1 (MANIFEST always 0 without flush) would
    start from the active segment, skipping earlier unflushed segments.

C3: Recover called manifest.Save after every recovery, advancing
    recoverySegmentID past segments that were still the only durable copy
    of their data (no SSTable flush yet). Next restart would filter those
    segments out and permanently lose the data.

Per design §3.2 line 280, recovery must not update MANIFEST; per line
604-06, CURRENT must not be used as recovery start. Both fixes are
required together — fixing C3 alone leaves C2's data-loss window open.

Changes:
- wal/recover.go: remove manifest.Save calls on both success and
  tail-repair paths; remove CURRENT fallback in resolveRecoverySegmentID.
  RecoveryResult.NextSegmentID is now in-memory only (consumed by DB.Open
  to seed the new WalWriter, but never persisted to MANIFEST).
- wal/recover_test.go: rewrite TestRecoverUpdatesManifest as
  TestRecoverDoesNotUpdateManifest; add TestRecoverPreservesExistingManifest,
  TestRecoverIdempotentClean, TestRecoverIdempotentAfterTruncation,
  TestRecoverIgnoresCurrentFallback.
- db_test.go: add TestOpenThreeTimesKeepsData (three opens to catch C3's
  second-restart data loss; single-segment to avoid unrelated C8 bug
  where segment_manager passes byte offset as startSequence).

Verified: each new test fails on pre-fix code and passes after the fix.
Full suite green including -race.

Audit context: docs/audit-3.2.md (with Oracle revisions from bg_ef425776
and bg_2e86d33b; C8 added). Plan: .omo/plans/fix-c2-c3-wal-recovery.md
(Momus + Oracle reviewed v1.2).
This commit is contained in:
dailz
2026-06-15 13:40:13 +08:00
parent e34de4acc9
commit 98fbac07f2
5 changed files with 1195 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wal/recover_test.go
+192 -14
View File
@@ -1,6 +1,7 @@
package wal
import (
"bytes"
"os"
"path/filepath"
"reflect"
@@ -124,34 +125,211 @@ func TestRecoverWithTailCorruption(t *testing.T) {
}
}
func TestRecoverUpdatesManifest(t *testing.T) {
func TestRecoverDoesNotUpdateManifest(t *testing.T) {
dir := t.TempDir()
// Write test data.
writeTestSegment(t, dir, 0, 50, [][]*WalEntry{
{makePutEntry("a", "b")},
{makePutEntry("c", "d")},
})
beforeMF, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load before recover: %v", err)
}
beforeExists := fileExists(t, filepath.Join(dir, "MANIFEST"))
replayer := &mockReplayer{}
if _, err := Recover(dir, replayer); err != nil {
t.Fatalf("Recover: %v", err)
}
afterMF, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load after recover: %v", err)
}
if afterMF.RecoverySegmentID != beforeMF.RecoverySegmentID {
t.Errorf("MANIFEST RecoverySegmentID changed: %d -> %d",
beforeMF.RecoverySegmentID, afterMF.RecoverySegmentID)
}
afterExists := fileExists(t, filepath.Join(dir, "MANIFEST"))
if beforeExists != afterExists {
t.Errorf("MANIFEST file existence changed: before=%v after=%v",
beforeExists, afterExists)
}
}
// Regression guard for C3: covers the case where MANIFEST already exists.
// The fresh-DB test above cannot catch accidental overwrites of an existing
// MANIFEST.
func TestRecoverPreservesExistingManifest(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 50, [][]*WalEntry{
{makePutEntry("a", "b")},
})
if err := manifest.Save(dir, 0); err != nil {
t.Fatalf("manifest.Save setup: %v", err)
}
beforeBytes, err := os.ReadFile(filepath.Join(dir, "MANIFEST"))
if err != nil {
t.Fatalf("ReadFile MANIFEST: %v", err)
}
replayer := &mockReplayer{}
if _, err := Recover(dir, replayer); err != nil {
t.Fatalf("Recover: %v", err)
}
afterBytes, err := os.ReadFile(filepath.Join(dir, "MANIFEST"))
if err != nil {
t.Fatalf("ReadFile MANIFEST after recover: %v", err)
}
if !bytes.Equal(beforeBytes, afterBytes) {
t.Errorf("MANIFEST bytes changed:\n before=%q\n after=%q",
string(beforeBytes), string(afterBytes))
}
}
// Regression guard for C3: design §3.2 line 280 requires recovery to be
// idempotent on a clean WAL (no MANIFEST side effects).
func TestRecoverIdempotentClean(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
{makePutEntry("k1", "v1")},
{makePutEntry("k2", "v2")},
})
replayer1 := &mockReplayer{}
result1, err := Recover(dir, replayer1)
if err != nil {
t.Fatalf("Recover (1st): %v", err)
}
replayer2 := &mockReplayer{}
result2, err := Recover(dir, replayer2)
if err != nil {
t.Fatalf("Recover (2nd): %v", err)
}
if result1.NextSequence != result2.NextSequence {
t.Errorf("NextSequence differs: %d vs %d", result1.NextSequence, result2.NextSequence)
}
if result1.NextSegmentID != result2.NextSegmentID {
t.Errorf("NextSegmentID differs: %d vs %d", result1.NextSegmentID, result2.NextSegmentID)
}
if result1.Truncated || result2.Truncated {
t.Errorf("Truncated should be false for clean WAL: r1=%v r2=%v",
result1.Truncated, result2.Truncated)
}
if !reflect.DeepEqual(replayer1.puts, replayer2.puts) {
t.Errorf("replayed puts differ:\n r1=%#v\n r2=%#v", replayer1.puts, replayer2.puts)
}
}
// Regression guard for C3: after the first recovery truncates a corrupted
// tail, the second recovery must observe stable state (Truncated=false, same
// NextSequence, same replayed entries).
func TestRecoverIdempotentAfterTruncation(t *testing.T) {
dir := t.TempDir()
filePath := writeTestSegment(t, dir, 0, 100, [][]*WalEntry{
{makePutEntry("good1", "before-corruption")},
{makePutEntry("good2", "also-before")},
})
appendFileBytes(t, filePath, []byte{0xDE, 0xAD, 0xBE, 0xEF})
replayer1 := &mockReplayer{}
result1, err := Recover(dir, replayer1)
if err != nil {
t.Fatalf("Recover (1st): %v", err)
}
if !result1.Truncated {
t.Fatal("1st Recover: Truncated = false, want true")
}
replayer2 := &mockReplayer{}
result2, err := Recover(dir, replayer2)
if err != nil {
t.Fatalf("Recover (2nd): %v", err)
}
if result2.Truncated {
t.Error("2nd Recover: Truncated = true, want false (tail already repaired)")
}
if result1.NextSequence != result2.NextSequence {
t.Errorf("NextSequence differs: %d vs %d", result1.NextSequence, result2.NextSequence)
}
if !reflect.DeepEqual(replayer1.puts, replayer2.puts) {
t.Errorf("replayed puts differ:\n r1=%#v\n r2=%#v", replayer1.puts, replayer2.puts)
}
}
// Regression guard for C2: design §3.2 line 604-06 forbids using CURRENT as
// recovery start. writeTestSegment uses NewSegmentWriter directly, which does
// NOT write CURRENT (only SegmentManager does), so we write CURRENT explicitly
// to simulate the Phase 1 default state.
func TestRecoverIgnoresCurrentFallback(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
{makePutEntry("seg0-k1", "v1")},
})
writeTestSegment(t, dir, 1, 1, [][]*WalEntry{
{makePutEntry("seg1-k1", "v1")},
})
writeTestSegment(t, dir, 2, 2, [][]*WalEntry{
{makePutEntry("seg2-k1", "v1")},
})
if err := manifest.WriteCurrent(dir, 2); err != nil {
t.Fatalf("WriteCurrent: %v", err)
}
currentSegID, ok := manifest.ReadCurrent(dir)
if !ok || currentSegID != 2 {
t.Fatalf("CURRENT setup wrong: segID=%d ok=%v", currentSegID, ok)
}
mf, err := manifest.Load(dir)
if err != nil {
t.Fatalf("Load: %v", err)
}
if mf.RecoverySegmentID != 0 {
t.Fatalf("MANIFEST.RecoverySegmentID = %d, want 0", mf.RecoverySegmentID)
}
replayer := &mockReplayer{}
result, err := Recover(dir, replayer)
if err != nil {
t.Fatalf("Recover: %v", err)
}
// Verify MANIFEST file exists and has correct content.
mf, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load after recover: %v", err)
wantPuts := []replayPut{
{key: "seg0-k1", value: "v1", seq: 0},
{key: "seg1-k1", value: "v1", seq: 1},
{key: "seg2-k1", value: "v1", seq: 2},
}
if mf.RecoverySegmentID != result.NextSegmentID {
t.Errorf("MANIFEST RecoverySegmentID = %d, want %d",
mf.RecoverySegmentID, result.NextSegmentID)
if !reflect.DeepEqual(replayer.puts, wantPuts) {
t.Errorf("puts = %#v, want %#v", replayer.puts, wantPuts)
}
// Verify MANIFEST file path.
manifestPath := filepath.Join(dir, "MANIFEST")
if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
t.Error("MANIFEST file should exist after Recover")
if result.NextSequence != 3 {
t.Errorf("NextSequence = %d, want 3", result.NextSequence)
}
if result.NextSegmentID != 3 {
t.Errorf("NextSegmentID = %d, want 3", result.NextSegmentID)
}
}
func fileExists(t *testing.T, path string) bool {
t.Helper()
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
t.Fatalf("stat %s: %v", path, err)
return false
}