feat(wal): implement recovery orchestrator with tail truncation
- wal/recover.go: top-level Recover() function that orchestrates scan + replay + truncation Reads MANIFEST/CURRENT for recovery segment ID, handles tail corruption by truncating - Comprehensive tests for empty dir, full flow, tail corruption, and MANIFEST updates - All tests pass with -race
This commit is contained in:
dailz
+28
-2
@@ -8,7 +8,7 @@
|
||||
"plan_name": "phase1-wal",
|
||||
"status": "active",
|
||||
"started_at": "2026-06-12T05:09:32.588Z",
|
||||
"updated_at": "2026-06-12T05:57:16.412Z",
|
||||
"updated_at": "2026-06-12T06:02:11.867Z",
|
||||
"session_ids": [
|
||||
"opencode:ses_145c3bae9ffeTB2zbsTym0Cev8"
|
||||
],
|
||||
@@ -107,6 +107,19 @@
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:57:16.412Z",
|
||||
"elapsed_ms": 23149
|
||||
},
|
||||
"todo:17": {
|
||||
"task_key": "todo:17",
|
||||
"task_label": "17",
|
||||
"task_title": "WAL recovery: tail truncation + main flow",
|
||||
"session_id": "opencode:ses_14597418effebiQE4kDHOCEW0E",
|
||||
"agent": "Sisyphus-Junior",
|
||||
"category": "unspecified-high",
|
||||
"updated_at": "2026-06-12T06:02:11.867Z",
|
||||
"started_at": "2026-06-12T06:01:55.154Z",
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T06:02:11.867Z",
|
||||
"elapsed_ms": 16713
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,7 +127,7 @@
|
||||
"active_plan": "/home/dailz/workspace/src/go-kv/.omo/plans/phase1-wal.md",
|
||||
"started_at": "2026-06-12T05:09:32.588Z",
|
||||
"status": "active",
|
||||
"updated_at": "2026-06-12T05:57:16.412Z",
|
||||
"updated_at": "2026-06-12T06:02:11.867Z",
|
||||
"session_ids": [
|
||||
"opencode:ses_145c3bae9ffeTB2zbsTym0Cev8"
|
||||
],
|
||||
@@ -213,6 +226,19 @@
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:57:16.412Z",
|
||||
"elapsed_ms": 23149
|
||||
},
|
||||
"todo:17": {
|
||||
"task_key": "todo:17",
|
||||
"task_label": "17",
|
||||
"task_title": "WAL recovery: tail truncation + main flow",
|
||||
"session_id": "opencode:ses_14597418effebiQE4kDHOCEW0E",
|
||||
"agent": "Sisyphus-Junior",
|
||||
"category": "unspecified-high",
|
||||
"updated_at": "2026-06-12T06:02:11.867Z",
|
||||
"started_at": "2026-06-12T06:01:55.154Z",
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T06:02:11.867Z",
|
||||
"elapsed_ms": 16713
|
||||
}
|
||||
},
|
||||
"agent": "atlas"
|
||||
|
||||
@@ -1287,7 +1287,7 @@ Max Concurrent: 6 (Wave 1b)
|
||||
|
||||
**Commit**: YES (with Tasks 15, 17)
|
||||
|
||||
- [ ] 17. WAL recovery: tail truncation + main flow
|
||||
- [x] 17. WAL recovery: tail truncation + main flow
|
||||
|
||||
**What to do**:
|
||||
- 创建 `wal/truncation.go`：尾部截断持久化
|
||||
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/dailz/go-kv/manifest"
|
||||
)
|
||||
|
||||
// RecoveryResult holds the outcome of a WAL recovery pass.
|
||||
type RecoveryResult struct {
|
||||
NextSequence uint64
|
||||
NextSegmentID uint64
|
||||
ReplayedEntries int
|
||||
Truncated bool
|
||||
TruncateError error // non-nil if tail corruption was found
|
||||
}
|
||||
|
||||
// Recover performs a full WAL recovery: reads the recovery checkpoint from
|
||||
// MANIFEST (or CURRENT), scans segments, replays entries, and handles tail
|
||||
// truncation. On success the MANIFEST is updated with the new recovery state.
|
||||
func Recover(dir string, replayer BatchReplayer) (*RecoveryResult, error) {
|
||||
if replayer == nil {
|
||||
return nil, fmt.Errorf("wal: recover: replayer is nil")
|
||||
}
|
||||
|
||||
// Step 1: Determine recovery segment ID from MANIFEST or CURRENT.
|
||||
recoverySegmentID, err := resolveRecoverySegmentID(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: recover: resolve segment id: %w", err)
|
||||
}
|
||||
|
||||
// Step 2: Scan and replay segments.
|
||||
nextSequence, err := RecoverFromSegments(dir, recoverySegmentID, replayer)
|
||||
if err != nil {
|
||||
if !IsTailCorruption(err) {
|
||||
return nil, fmt.Errorf("wal: recover: %w", err)
|
||||
}
|
||||
|
||||
// Step 3: Tail corruption — truncate the last segment and accept
|
||||
// partial data loss for Phase 1.
|
||||
result := &RecoveryResult{
|
||||
NextSequence: nextSequence,
|
||||
Truncated: true,
|
||||
TruncateError: err,
|
||||
}
|
||||
|
||||
// Determine nextSegmentID from scanned segments.
|
||||
segments, scanErr := ScanSegments(dir, recoverySegmentID)
|
||||
if scanErr != nil {
|
||||
return nil, fmt.Errorf("wal: recover: scan after tail corruption: %w", scanErr)
|
||||
}
|
||||
if len(segments) > 0 {
|
||||
result.NextSegmentID = segments[len(segments)-1].SegmentID + 1
|
||||
} else {
|
||||
result.NextSegmentID = recoverySegmentID
|
||||
}
|
||||
|
||||
// Truncate the last segment file to remove corrupted tail.
|
||||
if len(segments) > 0 {
|
||||
lastSeg := segments[len(segments)-1]
|
||||
validOffset, truncErr := findValidOffset(lastSeg.FilePath)
|
||||
if truncErr != nil {
|
||||
// Best-effort: record the truncation error but don't fail recovery.
|
||||
result.TruncateError = fmt.Errorf("%w (find valid offset: %v)", err, truncErr)
|
||||
} else if truncErr := truncateSegment(lastSeg.FilePath, validOffset); truncErr != nil {
|
||||
result.TruncateError = fmt.Errorf("%w (truncate: %v)", err, truncErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Count replayed entries by re-scanning the replayer state.
|
||||
// For Phase 1 we accept that ReplayedEntries may be approximate;
|
||||
// the replayer interface doesn't expose a count.
|
||||
result.ReplayedEntries = 0 // caller can inspect replayer directly
|
||||
|
||||
// Update MANIFEST with new recovery state.
|
||||
if saveErr := manifest.Save(dir, result.NextSegmentID); saveErr != nil {
|
||||
return nil, fmt.Errorf("wal: recover: save manifest after truncation: %w", saveErr)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Step 4: Successful recovery — compute result.
|
||||
segments, scanErr := ScanSegments(dir, recoverySegmentID)
|
||||
if scanErr != nil {
|
||||
return nil, fmt.Errorf("wal: recover: scan after replay: %w", scanErr)
|
||||
}
|
||||
|
||||
result := &RecoveryResult{
|
||||
NextSequence: nextSequence,
|
||||
NextSegmentID: recoverySegmentID,
|
||||
Truncated: false,
|
||||
}
|
||||
if len(segments) > 0 {
|
||||
result.NextSegmentID = segments[len(segments)-1].SegmentID + 1
|
||||
}
|
||||
|
||||
// Update MANIFEST with new recovery state.
|
||||
if saveErr := manifest.Save(dir, result.NextSegmentID); saveErr != nil {
|
||||
return nil, fmt.Errorf("wal: recover: save manifest: %w", saveErr)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// resolveRecoverySegmentID determines the starting segment ID for recovery.
|
||||
// It tries MANIFEST first, then falls back to CURRENT, then defaults to 0.
|
||||
func resolveRecoverySegmentID(dir string) (uint64, error) {
|
||||
mf, err := manifest.Load(dir)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
if mf.RecoverySegmentID > 0 {
|
||||
return mf.RecoverySegmentID, nil
|
||||
}
|
||||
|
||||
// MANIFEST had 0 (fresh DB or not yet written). Try CURRENT.
|
||||
if segID, ok := manifest.ReadCurrent(dir); ok {
|
||||
return segID, nil
|
||||
}
|
||||
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// truncateSegment truncates the file at filePath to validOffset bytes,
|
||||
// removing any corrupted data after that point.
|
||||
func truncateSegment(filePath string, validOffset int64) error {
|
||||
if validOffset < 0 {
|
||||
return fmt.Errorf("wal: truncate: invalid offset %d", validOffset)
|
||||
}
|
||||
return os.Truncate(filePath, validOffset)
|
||||
}
|
||||
|
||||
// findValidOffset parses a segment file and returns the byte offset of the
|
||||
// last valid record boundary. The offset includes the file header size.
|
||||
func findValidOffset(filePath string) (int64, error) {
|
||||
// Re-parse the file to find where valid records end.
|
||||
// We need to track the byte offset as we parse.
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open for offset scan: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err := f.Seek(WalFileHeaderSize, 0); err != nil {
|
||||
return 0, fmt.Errorf("seek past header: %w", err)
|
||||
}
|
||||
|
||||
validOffset := int64(WalFileHeaderSize)
|
||||
buf := make([]byte, WalBlockSize)
|
||||
|
||||
for {
|
||||
n, readErr := f.Read(buf)
|
||||
if readErr != nil {
|
||||
break
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
blockData := buf[:n]
|
||||
blockStartOffset := validOffset
|
||||
pos := 0
|
||||
|
||||
for pos < len(blockData) {
|
||||
remaining := len(blockData) - pos
|
||||
|
||||
if remaining < PhysicalRecordHeaderSize {
|
||||
// Check if remaining bytes are zero-padding.
|
||||
if isAllZeros(blockData[pos:]) {
|
||||
// Valid padding — update offset to end of last valid record.
|
||||
validOffset = blockStartOffset + int64(pos)
|
||||
}
|
||||
// Either way, we're done with this block.
|
||||
break
|
||||
}
|
||||
|
||||
if isAllZeros(blockData[pos : pos+PhysicalRecordHeaderSize]) {
|
||||
if isAllZeros(blockData[pos:]) {
|
||||
validOffset = blockStartOffset + int64(pos)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
rec, consumed, err := DecodePhysicalRecord(blockData[pos:])
|
||||
if err != nil {
|
||||
// Corruption starts here — offset is up to last valid record.
|
||||
validOffset = blockStartOffset + int64(pos)
|
||||
return validOffset, nil
|
||||
}
|
||||
|
||||
// Valid record found.
|
||||
validOffset = blockStartOffset + int64(pos+consumed)
|
||||
// Keep the rec reference alive so the compiler doesn't optimize it away.
|
||||
_ = rec
|
||||
pos += consumed
|
||||
}
|
||||
|
||||
if n < WalBlockSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return validOffset, nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/dailz/go-kv/manifest"
|
||||
)
|
||||
|
||||
func TestRecoverFromEmptyDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
replayer := &mockReplayer{}
|
||||
result, err := Recover(dir, replayer)
|
||||
if err != nil {
|
||||
t.Fatalf("Recover empty dir: %v", err)
|
||||
}
|
||||
if result.NextSequence != 0 {
|
||||
t.Errorf("NextSequence = %d, want 0", result.NextSequence)
|
||||
}
|
||||
if result.Truncated {
|
||||
t.Error("Truncated = true, want false")
|
||||
}
|
||||
if result.TruncateError != nil {
|
||||
t.Errorf("TruncateError = %v, want nil", result.TruncateError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverFullFlow(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Write test data: 2 batches across 1 segment.
|
||||
writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
|
||||
{makePutEntry("key1", "val1"), makeDeleteEntry("key2")},
|
||||
{makePutEntry("key3", "val3")},
|
||||
})
|
||||
|
||||
replayer := &mockReplayer{}
|
||||
result, err := Recover(dir, replayer)
|
||||
if err != nil {
|
||||
t.Fatalf("Recover: %v", err)
|
||||
}
|
||||
if result.NextSequence != 3 {
|
||||
t.Errorf("NextSequence = %d, want 3", result.NextSequence)
|
||||
}
|
||||
if result.NextSegmentID != 1 {
|
||||
t.Errorf("NextSegmentID = %d, want 1", result.NextSegmentID)
|
||||
}
|
||||
if result.Truncated {
|
||||
t.Error("Truncated = true, want false")
|
||||
}
|
||||
|
||||
wantPuts := []replayPut{
|
||||
{key: "key1", value: "val1", seq: 0},
|
||||
{key: "key3", value: "val3", seq: 2},
|
||||
}
|
||||
if !reflect.DeepEqual(replayer.puts, wantPuts) {
|
||||
t.Errorf("puts = %#v, want %#v", replayer.puts, wantPuts)
|
||||
}
|
||||
wantDeletes := []replayDelete{
|
||||
{key: "key2", seq: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(replayer.deletes, wantDeletes) {
|
||||
t.Errorf("deletes = %#v, want %#v", replayer.deletes, wantDeletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverWithTailCorruption(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Write valid batches, then corrupt the tail.
|
||||
filePath := writeTestSegment(t, dir, 0, 100, [][]*WalEntry{
|
||||
{makePutEntry("good1", "before-corruption")},
|
||||
{makePutEntry("good2", "also-before")},
|
||||
})
|
||||
appendFileBytes(t, filePath, []byte{0xDE, 0xAD, 0xBE, 0xEF})
|
||||
|
||||
replayer := &mockReplayer{}
|
||||
result, err := Recover(dir, replayer)
|
||||
if err != nil {
|
||||
t.Fatalf("Recover with tail corruption: %v", err)
|
||||
}
|
||||
if !result.Truncated {
|
||||
t.Fatal("Truncated = false, want true")
|
||||
}
|
||||
if result.TruncateError == nil {
|
||||
t.Fatal("TruncateError = nil, want non-nil")
|
||||
}
|
||||
if result.NextSequence != 102 {
|
||||
t.Errorf("NextSequence = %d, want 102", result.NextSequence)
|
||||
}
|
||||
if result.NextSegmentID != 1 {
|
||||
t.Errorf("NextSegmentID = %d, want 1", result.NextSegmentID)
|
||||
}
|
||||
|
||||
// Verify only the valid entries were replayed.
|
||||
wantPuts := []replayPut{
|
||||
{key: "good1", value: "before-corruption", seq: 100},
|
||||
{key: "good2", value: "also-before", seq: 101},
|
||||
}
|
||||
if !reflect.DeepEqual(replayer.puts, wantPuts) {
|
||||
t.Errorf("puts = %#v, want %#v", replayer.puts, wantPuts)
|
||||
}
|
||||
|
||||
// Verify the file was truncated — it should be smaller than before.
|
||||
fi, fiErr := os.Stat(filePath)
|
||||
if fiErr != nil {
|
||||
t.Fatalf("Stat truncated file: %v", fiErr)
|
||||
}
|
||||
if fi.Size() == 0 {
|
||||
t.Error("truncated file is empty")
|
||||
}
|
||||
|
||||
// Verify the truncated file still parses cleanly.
|
||||
replayer2 := &mockReplayer{}
|
||||
_, parseErr := ReplaySegmentFile(filePath, 100, replayer2)
|
||||
if parseErr != nil {
|
||||
t.Fatalf("replay after truncation: %v", parseErr)
|
||||
}
|
||||
if !reflect.DeepEqual(replayer2.puts, wantPuts) {
|
||||
t.Errorf("replay after truncation puts = %#v, want %#v", replayer2.puts, wantPuts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverUpdatesManifest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Write test data.
|
||||
writeTestSegment(t, dir, 0, 50, [][]*WalEntry{
|
||||
{makePutEntry("a", "b")},
|
||||
{makePutEntry("c", "d")},
|
||||
})
|
||||
|
||||
replayer := &mockReplayer{}
|
||||
result, err := Recover(dir, replayer)
|
||||
if err != nil {
|
||||
t.Fatalf("Recover: %v", err)
|
||||
}
|
||||
|
||||
// Verify MANIFEST file exists and has correct content.
|
||||
mf, err := manifest.Load(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("manifest.Load after recover: %v", err)
|
||||
}
|
||||
if mf.RecoverySegmentID != result.NextSegmentID {
|
||||
t.Errorf("MANIFEST RecoverySegmentID = %d, want %d",
|
||||
mf.RecoverySegmentID, result.NextSegmentID)
|
||||
}
|
||||
|
||||
// Verify MANIFEST file path.
|
||||
manifestPath := filepath.Join(dir, "MANIFEST")
|
||||
if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
|
||||
t.Error("MANIFEST file should exist after Recover")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user