dailz/go-kv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: stop WAL recovery from advancing MANIFEST or using CURRENT (C2+C3)

Browse Source 
Phase 1 default state had two data-loss paths in WAL recovery.

C2: resolveRecoverySegmentID fell back to CURRENT when MANIFEST=0.
    Since segment_manager writes CURRENT on every segment create/rotate,
    the first recovery in Phase 1 (MANIFEST always 0 without flush) would
    start from the active segment, skipping earlier unflushed segments.

C3: Recover called manifest.Save after every recovery, advancing
    recoverySegmentID past segments that were still the only durable copy
    of their data (no SSTable flush yet). Next restart would filter those
    segments out and permanently lose the data.

Per design §3.2 line 280, recovery must not update MANIFEST; per line
604-06, CURRENT must not be used as recovery start. Both fixes are
required together — fixing C3 alone leaves C2's data-loss window open.

Changes:
- wal/recover.go: remove manifest.Save calls on both success and
  tail-repair paths; remove CURRENT fallback in resolveRecoverySegmentID.
  RecoveryResult.NextSegmentID is now in-memory only (consumed by DB.Open
  to seed the new WalWriter, but never persisted to MANIFEST).
- wal/recover_test.go: rewrite TestRecoverUpdatesManifest as
  TestRecoverDoesNotUpdateManifest; add TestRecoverPreservesExistingManifest,
  TestRecoverIdempotentClean, TestRecoverIdempotentAfterTruncation,
  TestRecoverIgnoresCurrentFallback.
- db_test.go: add TestOpenThreeTimesKeepsData (three opens to catch C3's
  second-restart data loss; single-segment to avoid unrelated C8 bug
  where segment_manager passes byte offset as startSequence).

Verified: each new test fails on pre-fix code and passes after the fix.
Full suite green including -race.

Audit context: docs/audit-3.2.md (with Oracle revisions from bg_ef425776
and bg_2e86d33b; C8 added). Plan: .omo/plans/fix-c2-c3-wal-recovery.md
(Momus + Oracle reviewed v1.2).
This commit is contained in:
dailz
2026-06-15 13:40:13 +08:00
parent e34de4acc9
commit 98fbac07f2
5 changed files with 1195 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wal/recover.go
+12 -21
View File
@@ -24,7 +24,7 @@ func Recover(dir string, replayer BatchReplayer) (*RecoveryResult, error) {
return nil, fmt.Errorf("wal: recover: replayer is nil")
}
// Step 1: Determine recovery segment ID from MANIFEST or CURRENT.
// Step 1: Determine recovery segment ID from MANIFEST.
recoverySegmentID, err := resolveRecoverySegmentID(dir)
if err != nil {
return nil, fmt.Errorf("wal: recover: resolve segment id: %w", err)
@@ -73,10 +73,10 @@ func Recover(dir string, replayer BatchReplayer) (*RecoveryResult, error) {
// the replayer interface doesn't expose a count.
result.ReplayedEntries = 0 // caller can inspect replayer directly
// Update MANIFEST with new recovery state.
if saveErr := manifest.Save(dir, result.NextSegmentID); saveErr != nil {
return nil, fmt.Errorf("wal: recover: save manifest after truncation: %w", saveErr)
}
// Per design §3.2 line 280, recovery repair must NOT update MANIFEST.
// The truncated WAL state is persisted via ftruncate (see C5 for the
// remaining fsync gaps). MANIFEST can only advance via checkpoint
// (MemTable flush) in future phases.
return result, nil
}
@@ -96,31 +96,22 @@ func Recover(dir string, replayer BatchReplayer) (*RecoveryResult, error) {
result.NextSegmentID = segments[len(segments)-1].SegmentID + 1
}
// Update MANIFEST with new recovery state.
if saveErr := manifest.Save(dir, result.NextSegmentID); saveErr != nil {
return nil, fmt.Errorf("wal: recover: save manifest: %w", saveErr)
}
// Per design §3.2 line 280, recovery must NOT update MANIFEST.
// RecoveryResult.NextSegmentID is in-memory only, consumed by DB.Open to
// seed the new WalWriter. MANIFEST stays at its pre-recovery value.
return result, nil
}
// resolveRecoverySegmentID determines the starting segment ID for recovery.
// It tries MANIFEST first, then falls back to CURRENT, then defaults to 0.
// resolveRecoverySegmentID returns the recovery start segment ID from MANIFEST.
// MANIFEST is the only authoritative source of recovery start per design §3.2
// line 600-06. CURRENT is a write-side hint and must NOT be used here.
func resolveRecoverySegmentID(dir string) (uint64, error) {
mf, err := manifest.Load(dir)
if err != nil {
return 0, fmt.Errorf("load manifest: %w", err)
}
if mf.RecoverySegmentID > 0 {
return mf.RecoverySegmentID, nil
}
// MANIFEST had 0 (fresh DB or not yet written). Try CURRENT.
if segID, ok := manifest.ReadCurrent(dir); ok {
return segID, nil
}
return 0, nil
return mf.RecoverySegmentID, nil
}
// truncateSegment truncates the file at filePath to validOffset bytes,
wal/recover_test.go
+192 -14
View File
@@ -1,6 +1,7 @@
package wal
import (
"bytes"
"os"
"path/filepath"
"reflect"
@@ -124,34 +125,211 @@ func TestRecoverWithTailCorruption(t *testing.T) {
}
}
func TestRecoverUpdatesManifest(t *testing.T) {
func TestRecoverDoesNotUpdateManifest(t *testing.T) {
dir := t.TempDir()
// Write test data.
writeTestSegment(t, dir, 0, 50, [][]*WalEntry{
{makePutEntry("a", "b")},
{makePutEntry("c", "d")},
})
beforeMF, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load before recover: %v", err)
}
beforeExists := fileExists(t, filepath.Join(dir, "MANIFEST"))
replayer := &mockReplayer{}
if _, err := Recover(dir, replayer); err != nil {
t.Fatalf("Recover: %v", err)
}
afterMF, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load after recover: %v", err)
}
if afterMF.RecoverySegmentID != beforeMF.RecoverySegmentID {
t.Errorf("MANIFEST RecoverySegmentID changed: %d -> %d",
beforeMF.RecoverySegmentID, afterMF.RecoverySegmentID)
}
afterExists := fileExists(t, filepath.Join(dir, "MANIFEST"))
if beforeExists != afterExists {
t.Errorf("MANIFEST file existence changed: before=%v after=%v",
beforeExists, afterExists)
}
}
// Regression guard for C3: covers the case where MANIFEST already exists.
// The fresh-DB test above cannot catch accidental overwrites of an existing
// MANIFEST.
func TestRecoverPreservesExistingManifest(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 50, [][]*WalEntry{
{makePutEntry("a", "b")},
})
if err := manifest.Save(dir, 0); err != nil {
t.Fatalf("manifest.Save setup: %v", err)
}
beforeBytes, err := os.ReadFile(filepath.Join(dir, "MANIFEST"))
if err != nil {
t.Fatalf("ReadFile MANIFEST: %v", err)
}
replayer := &mockReplayer{}
if _, err := Recover(dir, replayer); err != nil {
t.Fatalf("Recover: %v", err)
}
afterBytes, err := os.ReadFile(filepath.Join(dir, "MANIFEST"))
if err != nil {
t.Fatalf("ReadFile MANIFEST after recover: %v", err)
}
if !bytes.Equal(beforeBytes, afterBytes) {
t.Errorf("MANIFEST bytes changed:\n before=%q\n after=%q",
string(beforeBytes), string(afterBytes))
}
}
// Regression guard for C3: design §3.2 line 280 requires recovery to be
// idempotent on a clean WAL (no MANIFEST side effects).
func TestRecoverIdempotentClean(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
{makePutEntry("k1", "v1")},
{makePutEntry("k2", "v2")},
})
replayer1 := &mockReplayer{}
result1, err := Recover(dir, replayer1)
if err != nil {
t.Fatalf("Recover (1st): %v", err)
}
replayer2 := &mockReplayer{}
result2, err := Recover(dir, replayer2)
if err != nil {
t.Fatalf("Recover (2nd): %v", err)
}
if result1.NextSequence != result2.NextSequence {
t.Errorf("NextSequence differs: %d vs %d", result1.NextSequence, result2.NextSequence)
}
if result1.NextSegmentID != result2.NextSegmentID {
t.Errorf("NextSegmentID differs: %d vs %d", result1.NextSegmentID, result2.NextSegmentID)
}
if result1.Truncated || result2.Truncated {
t.Errorf("Truncated should be false for clean WAL: r1=%v r2=%v",
result1.Truncated, result2.Truncated)
}
if !reflect.DeepEqual(replayer1.puts, replayer2.puts) {
t.Errorf("replayed puts differ:\n r1=%#v\n r2=%#v", replayer1.puts, replayer2.puts)
}
}
// Regression guard for C3: after the first recovery truncates a corrupted
// tail, the second recovery must observe stable state (Truncated=false, same
// NextSequence, same replayed entries).
func TestRecoverIdempotentAfterTruncation(t *testing.T) {
dir := t.TempDir()
filePath := writeTestSegment(t, dir, 0, 100, [][]*WalEntry{
{makePutEntry("good1", "before-corruption")},
{makePutEntry("good2", "also-before")},
})
appendFileBytes(t, filePath, []byte{0xDE, 0xAD, 0xBE, 0xEF})
replayer1 := &mockReplayer{}
result1, err := Recover(dir, replayer1)
if err != nil {
t.Fatalf("Recover (1st): %v", err)
}
if !result1.Truncated {
t.Fatal("1st Recover: Truncated = false, want true")
}
replayer2 := &mockReplayer{}
result2, err := Recover(dir, replayer2)
if err != nil {
t.Fatalf("Recover (2nd): %v", err)
}
if result2.Truncated {
t.Error("2nd Recover: Truncated = true, want false (tail already repaired)")
}
if result1.NextSequence != result2.NextSequence {
t.Errorf("NextSequence differs: %d vs %d", result1.NextSequence, result2.NextSequence)
}
if !reflect.DeepEqual(replayer1.puts, replayer2.puts) {
t.Errorf("replayed puts differ:\n r1=%#v\n r2=%#v", replayer1.puts, replayer2.puts)
}
}
// Regression guard for C2: design §3.2 line 604-06 forbids using CURRENT as
// recovery start. writeTestSegment uses NewSegmentWriter directly, which does
// NOT write CURRENT (only SegmentManager does), so we write CURRENT explicitly
// to simulate the Phase 1 default state.
func TestRecoverIgnoresCurrentFallback(t *testing.T) {
dir := t.TempDir()
writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
{makePutEntry("seg0-k1", "v1")},
})
writeTestSegment(t, dir, 1, 1, [][]*WalEntry{
{makePutEntry("seg1-k1", "v1")},
})
writeTestSegment(t, dir, 2, 2, [][]*WalEntry{
{makePutEntry("seg2-k1", "v1")},
})
if err := manifest.WriteCurrent(dir, 2); err != nil {
t.Fatalf("WriteCurrent: %v", err)
}
currentSegID, ok := manifest.ReadCurrent(dir)
if !ok || currentSegID != 2 {
t.Fatalf("CURRENT setup wrong: segID=%d ok=%v", currentSegID, ok)
}
mf, err := manifest.Load(dir)
if err != nil {
t.Fatalf("Load: %v", err)
}
if mf.RecoverySegmentID != 0 {
t.Fatalf("MANIFEST.RecoverySegmentID = %d, want 0", mf.RecoverySegmentID)
}
replayer := &mockReplayer{}
result, err := Recover(dir, replayer)
if err != nil {
t.Fatalf("Recover: %v", err)
}
// Verify MANIFEST file exists and has correct content.
mf, err := manifest.Load(dir)
if err != nil {
t.Fatalf("manifest.Load after recover: %v", err)
wantPuts := []replayPut{
{key: "seg0-k1", value: "v1", seq: 0},
{key: "seg1-k1", value: "v1", seq: 1},
{key: "seg2-k1", value: "v1", seq: 2},
}
if mf.RecoverySegmentID != result.NextSegmentID {
t.Errorf("MANIFEST RecoverySegmentID = %d, want %d",
mf.RecoverySegmentID, result.NextSegmentID)
if !reflect.DeepEqual(replayer.puts, wantPuts) {
t.Errorf("puts = %#v, want %#v", replayer.puts, wantPuts)
}
// Verify MANIFEST file path.
manifestPath := filepath.Join(dir, "MANIFEST")
if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
t.Error("MANIFEST file should exist after Recover")
if result.NextSequence != 3 {
t.Errorf("NextSequence = %d, want 3", result.NextSequence)
}
if result.NextSegmentID != 3 {
t.Errorf("NextSegmentID = %d, want 3", result.NextSegmentID)
}
}
func fileExists(t *testing.T, path string) bool {
t.Helper()
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
t.Fatalf("stat %s: %v", path, err)
return false
}