fix: persist WAL tail truncation per design protocol (C5+H8)
Recovery tail-truncation had two compounding bugs in wal/recover.go:
C5: truncateSegment only called os.Truncate. Missing per design §3.2
line 787-794:
- Step 2: fsync the truncated segment
- Step 3: delete empty trailing segments
- Step 4: fsync WAL directory
And all errors were swallowed into result.TruncateError with recovery
still returning success, violating design line 799: "若 ftruncate、
segment fsync、空 segment 删除或 WAL directory fsync 任一步失败,
recovery 必须报错,DB 不得进入可写状态".
H8: findValidOffset only checked physical record CRCs, ignoring the
FragmentCollector state machine. For a tail of First + Middle*
without Last, it returned the offset AFTER the last Middle fragment
instead of the last COMPLETE batch end. Result: residual half-batch
fragments caused repeated tail-corruption reports on every restart.
Changes:
- wal/recover.go:
- Add findLastCompleteBatchEnd: batch-aware offset finder using
FragmentCollector state machine. Handles block-boundary padding
correctly (continue across full-block padding, return on short-block).
- Add truncateAndPersist: 4-step protocol (ftruncate + fsync segment +
delete empty trailing + fsync dir). Any step failure is fatal.
- Add segmentFsyncFn (package-level var for test injection, same
pattern as C6's dirFsyncFn).
- Refactor Recover failure path: use new functions, hard-error on
truncation persist failure (was: swallow to TruncateError).
- TruncateError field semantics: informational only ("tail corruption
was detected and repair attempted"). Persist failures return error.
- Delete findValidOffset and truncateSegment (replaced).
- wal/recover_offset_test.go (new): 8 unit tests for
findLastCompleteBatchEnd covering clean/partial-tail/no-batch/
physical-corruption/partial-only/block-boundary-padding/non-zero-tail/
zero-tail cases. 5 unit tests for truncateAndPersist covering success/
ftruncate-fail/dir-fsync-fail/segment-fsync-fail/retry-after-failure.
- wal/recover_test.go: add TestRecoverPartialFragmentTailIdempotent
(H8 e2e regression: truncation point must be at last complete batch),
TestRecoverTruncationFailureFailsRecovery (C5 e2e regression: any
step failure fails Recover), TestRecoverInvalidBatchNotTruncatable
(design line 778-781: invalid batch content hard-fails, NOT truncatable).
Injection note: segmentFsyncFn and dirFsyncFn (from C6) are package-level
vars; tests that override either must not use t.Parallel().
Verified: each new test fails on pre-fix code by logical analysis and
passes after the fix. Full suite green including go test -race ./... .
Phase 1 simplification: emptyTrailingSegments is always nil in Phase 1
(truncated segment is always segments[last]). The parameter is kept in
truncateAndPersist's signature for forward compatibility with the C4 fix.
Audit context: docs/audit-3.2.md C5 and H8 (H8 Oracle-verified bg_ef425776).
This commit is contained in:
@@ -2,9 +2,11 @@ package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dailz/go-kv/manifest"
|
||||
@@ -333,3 +335,119 @@ func fileExists(t *testing.T, path string) bool {
|
||||
t.Fatalf("stat %s: %v", path, err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Regression guard for C5+H8: end-to-end recovery with partial fragment
|
||||
// tail must persist truncation at the last COMPLETE batch boundary
|
||||
// (H8), and the truncation must be persisted with all 4 steps (C5).
|
||||
// After repair, second recovery must not see corruption.
|
||||
func TestRecoverPartialFragmentTailIdempotent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
batchA := []*WalEntry{makePutEntry("key-A", "val-A")}
|
||||
batchB := []*WalEntry{makePutEntry("key-B", "val-B")}
|
||||
|
||||
filePath := writeTestSegment(t, dir, 0, 0, [][]*WalEntry{batchA, batchB})
|
||||
|
||||
// Compute exact byte offset where Batch B's Full record ends.
|
||||
// Layout: [header][Batch A Full record][Batch B Full record][padding to 32KB]
|
||||
encA, _ := EncodeWalBatch(0, batchA)
|
||||
encB, _ := EncodeWalBatch(1, batchB)
|
||||
endOfBatchB := int64(WalFileHeaderSize) +
|
||||
int64(PhysicalRecordHeaderSize+len(encA)) +
|
||||
int64(PhysicalRecordHeaderSize+len(encB))
|
||||
|
||||
fiBefore, _ := os.Stat(filePath)
|
||||
|
||||
// Append First + Middle* (no Last) to simulate partial fragment tail.
|
||||
appendFileBytes(t, filePath, EncodePhysicalRecord(RecFirst, []byte("first-fragment-payload")))
|
||||
appendFileBytes(t, filePath, EncodePhysicalRecord(RecMiddle, []byte("middle-fragment-payload")))
|
||||
|
||||
replayer1 := &mockReplayer{}
|
||||
result1, err := Recover(dir, replayer1)
|
||||
if err != nil {
|
||||
t.Fatalf("1st Recover: %v", err)
|
||||
}
|
||||
if !result1.Truncated {
|
||||
t.Fatal("1st Recover: Truncated = false, want true")
|
||||
}
|
||||
|
||||
fiAfter, _ := os.Stat(filePath)
|
||||
if fiAfter.Size() != endOfBatchB {
|
||||
t.Errorf("file size after truncation = %d, want %d (end of Batch B, H8)",
|
||||
fiAfter.Size(), endOfBatchB)
|
||||
}
|
||||
if fiAfter.Size() >= fiBefore.Size() {
|
||||
t.Errorf("file should shrink after truncation: before=%d after=%d",
|
||||
fiBefore.Size(), fiAfter.Size())
|
||||
}
|
||||
|
||||
replayer2 := &mockReplayer{}
|
||||
result2, err := Recover(dir, replayer2)
|
||||
if err != nil {
|
||||
t.Fatalf("2nd Recover: %v", err)
|
||||
}
|
||||
if result2.Truncated {
|
||||
t.Error("2nd Recover: Truncated = true, want false (truncation should be persisted)")
|
||||
}
|
||||
if result1.NextSequence != result2.NextSequence {
|
||||
t.Errorf("NextSequence differs: %d vs %d", result1.NextSequence, result2.NextSequence)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression guard for C5: any truncation persist step failure must
|
||||
// fail Recover, causing DB.Open to fail. No swallowing allowed.
|
||||
func TestRecoverTruncationFailureFailsRecovery(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
filePath := writeTestSegment(t, dir, 0, 0, [][]*WalEntry{
|
||||
{makePutEntry("key-A", "val-A")},
|
||||
})
|
||||
// Append corruption to trigger tail corruption path.
|
||||
appendFileBytes(t, filePath, []byte{0xDE, 0xAD, 0xBE, 0xEF})
|
||||
|
||||
// Inject dir fsync failure (Step 4 of truncateAndPersist).
|
||||
orig := dirFsyncFn
|
||||
dirFsyncFn = func(string) error { return errors.New("simulated dir fsync failure") }
|
||||
t.Cleanup(func() { dirFsyncFn = orig })
|
||||
|
||||
_, err := Recover(dir, &mockReplayer{})
|
||||
if err == nil {
|
||||
t.Fatal("expected Recover to fail when truncation persist fails")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "persist tail truncation") {
|
||||
t.Errorf("error should mention 'persist tail truncation', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression guard for design line 778-781: CRC-valid but batch-content-
|
||||
// invalid must hard-fail through DecodeWalBatch, NOT enter truncation path.
|
||||
func TestRecoverInvalidBatchNotTruncatable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Build segment with: physical records CRC-valid, but assembled batch
|
||||
// has invalid header (entryCount=0).
|
||||
filePath := filepath.Join(dir, "segment-0.wal")
|
||||
writeRawSegmentHeader(t, filePath)
|
||||
|
||||
// Construct an "invalid batch": WalBatchHeaderSize=18 bytes, with
|
||||
// entryCount=0 (invalid per ReplayBatch check at recovery.go).
|
||||
invalidBatch := make([]byte, WalBatchHeaderSize)
|
||||
// flags(2) + baseSequence(8) + entryCount(4)=0 + entriesSize(4)=0
|
||||
// All zeros, except entryCount=0 is invalid by itself.
|
||||
// Encode as Full physical record (CRC-valid).
|
||||
rec := EncodePhysicalRecord(RecFull, invalidBatch)
|
||||
appendFileBytes(t, filePath, rec)
|
||||
|
||||
_, err := Recover(dir, &mockReplayer{})
|
||||
if err == nil {
|
||||
t.Fatal("expected Recover to fail on invalid batch content")
|
||||
}
|
||||
// Should NOT mention truncation — must be a different error path.
|
||||
if strings.Contains(err.Error(), "truncat") {
|
||||
t.Errorf("error should not be about truncation; got: %v", err)
|
||||
}
|
||||
|
||||
// File must NOT have been truncated (size unchanged).
|
||||
fi, _ := os.Stat(filePath)
|
||||
if fi.Size() != int64(WalFileHeaderSize)+int64(len(rec)) {
|
||||
t.Errorf("file was truncated; size = %d, want %d",
|
||||
fi.Size(), int64(WalFileHeaderSize)+int64(len(rec)))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user