feat(wal): implement segment rotation, recovery scanner, and record parser
- wal/segment_manager.go: segment lifecycle with rotation at batch boundaries - wal/scanner.go: segment discovery, ordering, and continuity validation - wal/record_parser.go: block-level physical record parsing with tail corruption detection - Comprehensive tests for all modules, all pass with -race
This commit is contained in:
dailz
+28
-2
@@ -8,7 +8,7 @@
|
||||
"plan_name": "phase1-wal",
|
||||
"status": "active",
|
||||
"started_at": "2026-06-12T05:09:32.588Z",
|
||||
"updated_at": "2026-06-12T05:43:11.052Z",
|
||||
"updated_at": "2026-06-12T05:49:50.501Z",
|
||||
"session_ids": [
|
||||
"opencode:ses_145c3bae9ffeTB2zbsTym0Cev8"
|
||||
],
|
||||
@@ -81,6 +81,19 @@
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:43:11.052Z",
|
||||
"elapsed_ms": 32768
|
||||
},
|
||||
"todo:11": {
|
||||
"task_key": "todo:11",
|
||||
"task_label": "11",
|
||||
"task_title": "Segment rotation + durable-ready protocol",
|
||||
"session_id": "opencode:ses_145a44184ffeEK5zu36ABE7jHM",
|
||||
"agent": "Sisyphus-Junior",
|
||||
"category": "unspecified-high",
|
||||
"updated_at": "2026-06-12T05:49:50.501Z",
|
||||
"started_at": "2026-06-12T05:48:04.491Z",
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:49:50.501Z",
|
||||
"elapsed_ms": 106010
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,7 +101,7 @@
|
||||
"active_plan": "/home/dailz/workspace/src/go-kv/.omo/plans/phase1-wal.md",
|
||||
"started_at": "2026-06-12T05:09:32.588Z",
|
||||
"status": "active",
|
||||
"updated_at": "2026-06-12T05:43:11.052Z",
|
||||
"updated_at": "2026-06-12T05:49:50.501Z",
|
||||
"session_ids": [
|
||||
"opencode:ses_145c3bae9ffeTB2zbsTym0Cev8"
|
||||
],
|
||||
@@ -161,6 +174,19 @@
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:43:11.052Z",
|
||||
"elapsed_ms": 32768
|
||||
},
|
||||
"todo:11": {
|
||||
"task_key": "todo:11",
|
||||
"task_label": "11",
|
||||
"task_title": "Segment rotation + durable-ready protocol",
|
||||
"session_id": "opencode:ses_145a44184ffeEK5zu36ABE7jHM",
|
||||
"agent": "Sisyphus-Junior",
|
||||
"category": "unspecified-high",
|
||||
"updated_at": "2026-06-12T05:49:50.501Z",
|
||||
"started_at": "2026-06-12T05:48:04.491Z",
|
||||
"status": "completed",
|
||||
"ended_at": "2026-06-12T05:49:50.501Z",
|
||||
"elapsed_ms": 106010
|
||||
}
|
||||
},
|
||||
"agent": "atlas"
|
||||
|
||||
@@ -891,7 +891,7 @@ Max Concurrent: 6 (Wave 1b)
|
||||
**Commit**: YES (with Task 11)
|
||||
- Message: `feat(wal): implement segment writer with rotation`
|
||||
|
||||
- [ ] 11. Segment rotation + durable-ready protocol
|
||||
- [x] 11. Segment rotation + durable-ready protocol
|
||||
|
||||
**What to do**:
|
||||
- 创建 `wal/segment_manager.go`：管理 segment 的轮转和生命周期
|
||||
@@ -1180,7 +1180,7 @@ Max Concurrent: 6 (Wave 1b)
|
||||
**Commit**: YES
|
||||
- Message: `feat(memtable): integrate arena+skiplist with publish/abort`
|
||||
|
||||
- [ ] 15. WAL recovery: scanner + record parser
|
||||
- [x] 15. WAL recovery: scanner + record parser
|
||||
|
||||
**What to do**:
|
||||
- 创建 `wal/scanner.go`：`SegmentScanner` — 从 WAL 目录扫描 segment 文件
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// TailCorruptionError indicates that the WAL tail contains corrupt data
|
||||
// (bad CRC, unexpected non-zero padding bytes, etc.). Recovery may safely
|
||||
// truncate at the last valid record.
|
||||
type TailCorruptionError struct {
|
||||
Offset int
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *TailCorruptionError) Error() string {
|
||||
return fmt.Sprintf("wal: tail corruption at offset %d: %v", e.Offset, e.Err)
|
||||
}
|
||||
|
||||
func (e *TailCorruptionError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// IsTailCorruption reports whether err indicates tail corruption in a WAL
|
||||
// segment. Callers may safely truncate the segment at the last valid record.
|
||||
func IsTailCorruption(err error) bool {
|
||||
var tce *TailCorruptionError
|
||||
return errors.As(err, &tce)
|
||||
}
|
||||
|
||||
// ParseBlock parses physical records from a single block of raw bytes.
|
||||
// The block is typically WalBlockSize (32 KB) bytes, but the last block of a
|
||||
// segment may be shorter. Trailing bytes after the last record must be all
|
||||
// zeros (padding); non-zero trailing bytes produce a TailCorruptionError.
|
||||
func ParseBlock(data []byte) ([]*PhysicalRecord, error) {
|
||||
var records []*PhysicalRecord
|
||||
pos := 0
|
||||
|
||||
for pos < len(data) {
|
||||
remaining := len(data) - pos
|
||||
|
||||
// If fewer than PhysicalRecordHeaderSize bytes remain, they must be
|
||||
// zero-padding.
|
||||
if remaining < PhysicalRecordHeaderSize {
|
||||
tail := data[pos:]
|
||||
if !isAllZeros(tail) {
|
||||
return records[:len(records):len(records)], &TailCorruptionError{
|
||||
Offset: pos,
|
||||
Err: fmt.Errorf("non-zero padding bytes in tail (%d bytes)", len(tail)),
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Check for zero-filled header (preallocated / unwritten space).
|
||||
if isAllZeros(data[pos : pos+PhysicalRecordHeaderSize]) {
|
||||
// Verify rest of block is also zeros.
|
||||
if !isAllZeros(data[pos:]) {
|
||||
return records[:len(records):len(records)], &TailCorruptionError{
|
||||
Offset: pos,
|
||||
Err: errors.New("zero header but non-zero bytes follow"),
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
rec, consumed, err := DecodePhysicalRecord(data[pos:])
|
||||
if err != nil {
|
||||
return records[:len(records):len(records)], &TailCorruptionError{
|
||||
Offset: pos,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
records = append(records, rec)
|
||||
pos += consumed
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// ParseRecordsFromFile opens a WAL segment file, skips the file header, reads
|
||||
// blocks sequentially, and returns all physical records in order. Short final
|
||||
// blocks are handled correctly.
|
||||
func ParseRecordsFromFile(filePath string) ([]*PhysicalRecord, error) {
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: parse records: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Skip file header.
|
||||
if _, err := f.Seek(WalFileHeaderSize, 0); err != nil {
|
||||
return nil, fmt.Errorf("wal: seek past header: %w", err)
|
||||
}
|
||||
|
||||
var allRecords []*PhysicalRecord
|
||||
buf := make([]byte, WalBlockSize)
|
||||
|
||||
for {
|
||||
n, readErr := f.Read(buf)
|
||||
if readErr != nil {
|
||||
if errors.Is(readErr, os.ErrClosed) {
|
||||
return nil, fmt.Errorf("wal: file closed during read: %w", readErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
blockData := buf[:n]
|
||||
recs, err := ParseBlock(blockData)
|
||||
if err != nil {
|
||||
// Return records collected so far along with the error.
|
||||
return allRecords, err
|
||||
}
|
||||
allRecords = append(allRecords, recs...)
|
||||
|
||||
// If we got a short block, this was the last one.
|
||||
if n < WalBlockSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return allRecords, nil
|
||||
}
|
||||
|
||||
func isAllZeros(data []byte) bool {
|
||||
return bytes.Count(data, []byte{0}) == len(data)
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBlockSingleRecord(t *testing.T) {
|
||||
payload := []byte("hello world")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
block := make([]byte, WalBlockSize)
|
||||
copy(block, encoded)
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseBlock: %v", err)
|
||||
}
|
||||
if len(recs) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(recs))
|
||||
}
|
||||
if recs[0].Type != RecFull {
|
||||
t.Errorf("Type = %d, want RecFull(%d)", recs[0].Type, RecFull)
|
||||
}
|
||||
if string(recs[0].Payload) != string(payload) {
|
||||
t.Errorf("Payload = %q, want %q", recs[0].Payload, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockMultipleRecords(t *testing.T) {
|
||||
payloads := [][]byte{
|
||||
[]byte("first"),
|
||||
[]byte("second"),
|
||||
[]byte("third"),
|
||||
}
|
||||
|
||||
block := make([]byte, WalBlockSize)
|
||||
offset := 0
|
||||
for i, p := range payloads {
|
||||
rec := EncodePhysicalRecord(RecFull, p)
|
||||
copy(block[offset:], rec)
|
||||
offset += len(rec)
|
||||
if offset > WalBlockSize {
|
||||
t.Fatalf("record %d overflows block", i)
|
||||
}
|
||||
}
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseBlock: %v", err)
|
||||
}
|
||||
if len(recs) != 3 {
|
||||
t.Fatalf("expected 3 records, got %d", len(recs))
|
||||
}
|
||||
for i, want := range payloads {
|
||||
if string(recs[i].Payload) != string(want) {
|
||||
t.Errorf("rec[%d].Payload = %q, want %q", i, recs[i].Payload, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockWithPadding(t *testing.T) {
|
||||
payload := []byte("data")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
// Place record at offset 0, then fill rest with zeros.
|
||||
block := make([]byte, WalBlockSize)
|
||||
copy(block, encoded)
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseBlock: %v", err)
|
||||
}
|
||||
if len(recs) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(recs))
|
||||
}
|
||||
if string(recs[0].Payload) != string(payload) {
|
||||
t.Errorf("Payload = %q, want %q", recs[0].Payload, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockCorruptPadding(t *testing.T) {
|
||||
payload := []byte("data")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
block := make([]byte, WalBlockSize)
|
||||
copy(block, encoded)
|
||||
// Write non-zero byte in the trailing padding area.
|
||||
block[len(encoded)+3] = 0xFF
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err == nil {
|
||||
t.Fatal("expected tail corruption error")
|
||||
}
|
||||
if !IsTailCorruption(err) {
|
||||
t.Errorf("expected TailCorruptionError, got: %v", err)
|
||||
}
|
||||
// Should still return records parsed before the corruption.
|
||||
if len(recs) != 1 {
|
||||
t.Errorf("expected 1 record before corruption, got %d", len(recs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockBadCRC(t *testing.T) {
|
||||
payload := []byte("data")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
block := make([]byte, WalBlockSize)
|
||||
copy(block, encoded)
|
||||
// Corrupt a payload byte.
|
||||
block[PhysicalRecordHeaderSize+1] ^= 0xFF
|
||||
|
||||
_, err := ParseBlock(block)
|
||||
if err == nil {
|
||||
t.Fatal("expected CRC mismatch error")
|
||||
}
|
||||
if !IsTailCorruption(err) {
|
||||
t.Errorf("expected TailCorruptionError, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockShortBlock(t *testing.T) {
|
||||
payload := []byte("short block data")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
// Simulate a short last block (less than WalBlockSize).
|
||||
block := make([]byte, len(encoded)+10) // extra trailing zeros
|
||||
copy(block, encoded)
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseBlock short block: %v", err)
|
||||
}
|
||||
if len(recs) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(recs))
|
||||
}
|
||||
if string(recs[0].Payload) != string(payload) {
|
||||
t.Errorf("Payload = %q, want %q", recs[0].Payload, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockEmpty(t *testing.T) {
|
||||
block := make([]byte, 64) // all zeros
|
||||
|
||||
recs, err := ParseBlock(block)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseBlock empty: %v", err)
|
||||
}
|
||||
if len(recs) != 0 {
|
||||
t.Fatalf("expected 0 records, got %d", len(recs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBlockZeroHeaderWithNonZeroTail(t *testing.T) {
|
||||
block := make([]byte, WalBlockSize)
|
||||
// First 7 bytes are zero (valid zero header), but byte at offset 8 is non-zero.
|
||||
block[8] = 0x42
|
||||
|
||||
_, err := ParseBlock(block)
|
||||
if err == nil {
|
||||
t.Fatal("expected tail corruption error for zero header + non-zero tail")
|
||||
}
|
||||
if !IsTailCorruption(err) {
|
||||
t.Errorf("expected TailCorruptionError, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTailCorruptionFalse(t *testing.T) {
|
||||
if IsTailCorruption(nil) {
|
||||
t.Error("IsTailCorruption(nil) = true, want false")
|
||||
}
|
||||
if IsTailCorruption(os.ErrNotExist) {
|
||||
t.Error("IsTailCorruption(ErrNotExist) = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRecordsFromFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "segment-0.wal")
|
||||
|
||||
// Create a segment file with header + one full block containing 2 records.
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hdr := &WalFileHeader{
|
||||
SegmentID: 0,
|
||||
StartSequence: 0,
|
||||
BlockSize: WalBlockSize,
|
||||
}
|
||||
encoded := EncodeWalHeader(hdr)
|
||||
if _, err := f.Write(encoded[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
block := make([]byte, WalBlockSize)
|
||||
payload1 := []byte("record-one")
|
||||
payload2 := []byte("record-two")
|
||||
offset := 0
|
||||
rec1 := EncodePhysicalRecord(RecFull, payload1)
|
||||
copy(block[offset:], rec1)
|
||||
offset += len(rec1)
|
||||
rec2 := EncodePhysicalRecord(RecFull, payload2)
|
||||
copy(block[offset:], rec2)
|
||||
|
||||
if _, err := f.Write(block); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
recs, err := ParseRecordsFromFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseRecordsFromFile: %v", err)
|
||||
}
|
||||
if len(recs) != 2 {
|
||||
t.Fatalf("expected 2 records, got %d", len(recs))
|
||||
}
|
||||
if string(recs[0].Payload) != string(payload1) {
|
||||
t.Errorf("rec[0].Payload = %q, want %q", recs[0].Payload, payload1)
|
||||
}
|
||||
if string(recs[1].Payload) != string(payload2) {
|
||||
t.Errorf("rec[1].Payload = %q, want %q", recs[1].Payload, payload2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRecordsFromFileShortLastBlock(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "segment-0.wal")
|
||||
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hdr := &WalFileHeader{
|
||||
SegmentID: 0,
|
||||
StartSequence: 0,
|
||||
BlockSize: WalBlockSize,
|
||||
}
|
||||
encoded := EncodeWalHeader(hdr)
|
||||
if _, err := f.Write(encoded[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write a partial block (just one record, no full 32KB).
|
||||
payload := []byte("short")
|
||||
rec := EncodePhysicalRecord(RecFull, payload)
|
||||
if _, err := f.Write(rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
recs, err := ParseRecordsFromFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseRecordsFromFile: %v", err)
|
||||
}
|
||||
if len(recs) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(recs))
|
||||
}
|
||||
if string(recs[0].Payload) != string(payload) {
|
||||
t.Errorf("Payload = %q, want %q", recs[0].Payload, payload)
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SegmentInfo holds metadata about a WAL segment file.
|
||||
type SegmentInfo struct {
|
||||
FilePath string
|
||||
SegmentID uint64
|
||||
StartSequence uint64
|
||||
}
|
||||
|
||||
// ParseSegmentFilename extracts the segment ID from a filename matching the
|
||||
// pattern "segment-{N}.wal". Returns (N, true) on match, (0, false) otherwise.
|
||||
func ParseSegmentFilename(name string) (segmentID uint64, ok bool) {
|
||||
if !strings.HasPrefix(name, "segment-") || !strings.HasSuffix(name, ".wal") {
|
||||
return 0, false
|
||||
}
|
||||
// Strip "segment-" prefix and ".wal" suffix.
|
||||
middle := name[len("segment-") : len(name)-len(".wal")]
|
||||
if len(middle) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
id, err := strconv.ParseUint(middle, 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrSegmentGap indicates missing WAL segment(s) in the expected sequence.
|
||||
ErrSegmentGap = errors.New("wal: gap detected in segment sequence")
|
||||
)
|
||||
|
||||
// ScanSegments discovers WAL segment files in dir, filters those with
|
||||
// SegmentID >= recoverySegmentID, validates file headers, and returns them
|
||||
// sorted by SegmentID ascending. Returns an error if a gap is detected in the
|
||||
// segment ID sequence.
|
||||
func ScanSegments(dir string, recoverySegmentID uint64) ([]*SegmentInfo, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: scan segments: %w", err)
|
||||
}
|
||||
|
||||
var candidates []*SegmentInfo
|
||||
for _, ent := range entries {
|
||||
if ent.IsDir() {
|
||||
continue
|
||||
}
|
||||
segID, ok := ParseSegmentFilename(ent.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if segID < recoverySegmentID {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, &SegmentInfo{
|
||||
FilePath: filepath.Join(dir, ent.Name()),
|
||||
SegmentID: segID,
|
||||
})
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Sort by SegmentID ascending.
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
return candidates[i].SegmentID < candidates[j].SegmentID
|
||||
})
|
||||
|
||||
// Decode each file header to populate StartSequence and validate.
|
||||
for _, si := range candidates {
|
||||
hdr, err := readFileHeader(si.FilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: segment %d: %w", si.SegmentID, err)
|
||||
}
|
||||
si.StartSequence = hdr.StartSequence
|
||||
}
|
||||
|
||||
// Verify continuity: segment IDs must form a consecutive sequence
|
||||
// starting from recoverySegmentID.
|
||||
for i, si := range candidates {
|
||||
expected := recoverySegmentID + uint64(i)
|
||||
if si.SegmentID != expected {
|
||||
return nil, fmt.Errorf("%w: expected segment %d, found %d",
|
||||
ErrSegmentGap, expected, si.SegmentID)
|
||||
}
|
||||
}
|
||||
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// readFileHeader opens the file, reads the header portion, and decodes it.
|
||||
func readFileHeader(path string) (*WalFileHeader, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hdrBuf := make([]byte, WalFileHeaderSize)
|
||||
n, err := f.Read(hdrBuf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read header: %w", err)
|
||||
}
|
||||
if n < WalFileHeaderSize {
|
||||
return nil, fmt.Errorf("read header: got %d bytes, need %d", n, WalFileHeaderSize)
|
||||
}
|
||||
return DecodeWalHeader(hdrBuf)
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// writeSegmentFile creates a minimal WAL segment file in dir with the given
|
||||
// segment ID and start sequence.
|
||||
func writeSegmentFile(t *testing.T, dir string, segmentID, startSequence uint64) {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, fmt.Sprintf("segment-%d.wal", segmentID))
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create segment file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hdr := &WalFileHeader{
|
||||
SegmentID: segmentID,
|
||||
StartSequence: startSequence,
|
||||
BlockSize: WalBlockSize,
|
||||
}
|
||||
encoded := EncodeWalHeader(hdr)
|
||||
if _, err := f.Write(encoded[:]); err != nil {
|
||||
t.Fatalf("write header: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSegmentFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
wantID uint64
|
||||
wantOK bool
|
||||
}{
|
||||
{"segment-0.wal", 0, true},
|
||||
{"segment-1.wal", 1, true},
|
||||
{"segment-42.wal", 42, true},
|
||||
{"segment-18446744073709551615.wal", 18446744073709551615, true},
|
||||
{"segment-.wal", 0, false}, // no digits
|
||||
{"segment-abc.wal", 0, false}, // non-numeric
|
||||
{"segment-1.wal.bak", 0, false}, // extra suffix
|
||||
{"foo-segment-1.wal", 0, false}, // extra prefix
|
||||
{" Segment-1.wal", 0, false}, // space prefix
|
||||
{"segment--1.wal", 0, false}, // negative sign
|
||||
{"", 0, false},
|
||||
{"readme.md", 0, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotID, gotOK := ParseSegmentFilename(tt.name)
|
||||
if gotOK != tt.wantOK {
|
||||
t.Errorf("ParseSegmentFilename(%q) ok = %v, want %v", tt.name, gotOK, tt.wantOK)
|
||||
}
|
||||
if gotOK && gotID != tt.wantID {
|
||||
t.Errorf("ParseSegmentFilename(%q) id = %d, want %d", tt.name, gotID, tt.wantID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSegmentsOrdered(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeSegmentFile(t, dir, 0, 100)
|
||||
writeSegmentFile(t, dir, 1, 200)
|
||||
writeSegmentFile(t, dir, 2, 300)
|
||||
|
||||
segs, err := ScanSegments(dir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ScanSegments: %v", err)
|
||||
}
|
||||
if len(segs) != 3 {
|
||||
t.Fatalf("expected 3 segments, got %d", len(segs))
|
||||
}
|
||||
|
||||
for i, want := range []struct {
|
||||
id uint64
|
||||
seq uint64
|
||||
}{
|
||||
{0, 100}, {1, 200}, {2, 300},
|
||||
} {
|
||||
if segs[i].SegmentID != want.id {
|
||||
t.Errorf("seg[%d].SegmentID = %d, want %d", i, segs[i].SegmentID, want.id)
|
||||
}
|
||||
if segs[i].StartSequence != want.seq {
|
||||
t.Errorf("seg[%d].StartSequence = %d, want %d", i, segs[i].StartSequence, want.seq)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSegmentsFiltersBelowRecoveryID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for i := uint64(0); i < 5; i++ {
|
||||
writeSegmentFile(t, dir, i, i*100)
|
||||
}
|
||||
|
||||
segs, err := ScanSegments(dir, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("ScanSegments: %v", err)
|
||||
}
|
||||
if len(segs) != 3 {
|
||||
t.Fatalf("expected 3 segments, got %d", len(segs))
|
||||
}
|
||||
|
||||
wantIDs := []uint64{2, 3, 4}
|
||||
for i, s := range segs {
|
||||
if s.SegmentID != wantIDs[i] {
|
||||
t.Errorf("seg[%d].SegmentID = %d, want %d", i, s.SegmentID, wantIDs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSegmentsGapDetection(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeSegmentFile(t, dir, 0, 0)
|
||||
writeSegmentFile(t, dir, 1, 100)
|
||||
// segment 2 is missing
|
||||
writeSegmentFile(t, dir, 3, 300)
|
||||
|
||||
_, err := ScanSegments(dir, 0)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for gap in segment sequence")
|
||||
}
|
||||
if !errors.Is(err, ErrSegmentGap) {
|
||||
t.Errorf("expected ErrSegmentGap, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSegmentsEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
segs, err := ScanSegments(dir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ScanSegments empty dir: %v", err)
|
||||
}
|
||||
if len(segs) != 0 {
|
||||
t.Fatalf("expected 0 segments, got %d", len(segs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSegmentsIgnoresNonWalFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeSegmentFile(t, dir, 0, 0)
|
||||
// Create non-WAL files that should be ignored.
|
||||
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("hi"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "segment-abc.wal"), []byte("nope"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
segs, err := ScanSegments(dir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ScanSegments: %v", err)
|
||||
}
|
||||
if len(segs) != 1 {
|
||||
t.Fatalf("expected 1 segment, got %d", len(segs))
|
||||
}
|
||||
if segs[0].SegmentID != 0 {
|
||||
t.Errorf("SegmentID = %d, want 0", segs[0].SegmentID)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/dailz/go-kv/config"
|
||||
"github.com/dailz/go-kv/manifest"
|
||||
)
|
||||
|
||||
// SegmentManager manages the lifecycle of WAL segment files, handling segment
|
||||
// rotation when the active segment runs out of payload capacity. A batch is
|
||||
// never split across segments — if it doesn't fit, a new segment is created
|
||||
// first and the entire batch is written there.
|
||||
type SegmentManager struct {
|
||||
dir string
|
||||
cfg *config.WalConfig
|
||||
active *SegmentWriter // currently active segment writer
|
||||
nextSegID uint64 // next segment ID to allocate
|
||||
}
|
||||
|
||||
// NewSegmentManager creates a new SegmentManager and its first segment file.
|
||||
// It creates the directory if needed, writes the initial segment, and updates
|
||||
// the CURRENT file to point to it.
|
||||
func NewSegmentManager(
|
||||
dir string,
|
||||
startSegmentID uint64,
|
||||
startSequence uint64,
|
||||
cfg *config.WalConfig,
|
||||
) (*SegmentManager, error) {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("wal: create segment directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
sw, err := NewSegmentWriter(dir, startSegmentID, startSequence, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: create initial segment: %w", err)
|
||||
}
|
||||
|
||||
sm := &SegmentManager{
|
||||
dir: dir,
|
||||
cfg: cfg,
|
||||
active: sw,
|
||||
nextSegID: startSegmentID + 1,
|
||||
}
|
||||
|
||||
// Best-effort CURRENT file update.
|
||||
_ = manifest.WriteCurrent(dir, sw.SegmentID())
|
||||
|
||||
return sm, nil
|
||||
}
|
||||
|
||||
// AppendBatch writes an encoded batch to the active segment. If the batch does
|
||||
// not fit in the remaining payload space (with worst-case physical record
|
||||
// overhead), the manager rotates to a fresh segment first so the entire batch
|
||||
// lands in one segment.
|
||||
func (sm *SegmentManager) AppendBatch(encodedBatch []byte) error {
|
||||
// Calculate the worst-case on-disk size for this batch:
|
||||
// len(encodedBatch) + at least one physical record header + block padding margin
|
||||
// This is a conservative upper bound. The actual overhead may be less due to
|
||||
// block alignment, but we must guarantee the batch won't exceed MaxSegmentSize.
|
||||
worstCaseSize := uint64(len(encodedBatch)) + uint64(PhysicalRecordHeaderSize) + uint64(PhysicalRecordHeaderSize)
|
||||
|
||||
if sm.active.RemainingPayload() < worstCaseSize {
|
||||
if err := sm.rotate(sm.active.CurrentOffset()); err != nil {
|
||||
return fmt.Errorf("wal: rotate segment: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return sm.active.AppendBatch(encodedBatch)
|
||||
}
|
||||
|
||||
// rotate closes the current segment and creates a new one. The CURRENT file is
|
||||
// updated on a best-effort basis — a failure is logged but does not prevent
|
||||
// the rotation from succeeding.
|
||||
func (sm *SegmentManager) rotate(newStartSequence uint64) error {
|
||||
if err := sm.active.Close(); err != nil {
|
||||
return fmt.Errorf("wal: close segment %d: %w", sm.active.SegmentID(), err)
|
||||
}
|
||||
|
||||
sw, err := NewSegmentWriter(sm.dir, sm.nextSegID, newStartSequence, sm.cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: create segment %d: %w", sm.nextSegID, err)
|
||||
}
|
||||
|
||||
sm.nextSegID++
|
||||
sm.active = sw
|
||||
|
||||
// Best-effort CURRENT file update — failure must not fail the write.
|
||||
_ = manifest.WriteCurrent(sm.dir, sw.SegmentID())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveSegmentID returns the segment ID of the currently active segment.
|
||||
func (sm *SegmentManager) ActiveSegmentID() uint64 {
|
||||
return sm.active.SegmentID()
|
||||
}
|
||||
|
||||
// RemainingPayload returns the number of bytes that can still be written to
|
||||
// the active segment.
|
||||
func (sm *SegmentManager) RemainingPayload() uint64 {
|
||||
return sm.active.RemainingPayload()
|
||||
}
|
||||
|
||||
// Sync flushes the active segment to durable storage.
|
||||
func (sm *SegmentManager) Sync() error {
|
||||
return sm.active.Sync()
|
||||
}
|
||||
|
||||
// Close flushes and closes the active segment.
|
||||
func (sm *SegmentManager) Close() error {
|
||||
return sm.active.Close()
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/dailz/go-kv/config"
|
||||
"github.com/dailz/go-kv/manifest"
|
||||
)
|
||||
|
||||
// tinyWalConfig returns a config with a very small MaxSegmentSize to force
|
||||
// quick rotation. The minimum is derived from config.Validate: we need enough
|
||||
// room for the file header, batch header, physical record overhead, and block
|
||||
// padding. We use 256 bytes which is well above the minimum for default
|
||||
// block/batch settings.
|
||||
func tinyWalConfig() *config.WalConfig {
|
||||
cfg := config.Defaults()
|
||||
// Use a small segment size to force rotation quickly.
|
||||
// MaxSegmentSize must be > WalFileHeaderSize (32) and pass Validate().
|
||||
// With defaults, minimum is around 4MB+overhead, so we must also reduce
|
||||
// MaxBatchSize and BlockSize to make a small segment valid.
|
||||
cfg.BlockSize = 512
|
||||
cfg.MaxBatchSize = 64 // very small batches
|
||||
cfg.MaxBatchEntries = 5
|
||||
cfg.MaxKeyBytes = 16
|
||||
cfg.MaxInlineValue = 16
|
||||
cfg.MaxSegmentSize = 512 // small enough to trigger rotation with a few writes
|
||||
return &cfg
|
||||
}
|
||||
|
||||
func TestSegmentManagerCreation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := testWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
// Verify segment-0.wal exists.
|
||||
expected := filepath.Join(dir, "segment-0.wal")
|
||||
if _, err := os.Stat(expected); err != nil {
|
||||
t.Errorf("segment file %q should exist: %v", expected, err)
|
||||
}
|
||||
|
||||
if sm.ActiveSegmentID() != 0 {
|
||||
t.Errorf("ActiveSegmentID = %d, want 0", sm.ActiveSegmentID())
|
||||
}
|
||||
|
||||
// Verify CURRENT file points to segment-0.
|
||||
segID, ok := manifest.ReadCurrent(dir)
|
||||
if !ok {
|
||||
t.Fatal("ReadCurrent: expected CURRENT file to exist")
|
||||
}
|
||||
if segID != 0 {
|
||||
t.Errorf("CURRENT segment ID = %d, want 0", segID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentManagerRotation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := tinyWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
// Write small batches until rotation occurs.
|
||||
// Each batch is a minimal encoded WAL batch: just a small payload.
|
||||
// We'll write enough to exhaust the tiny segment.
|
||||
batch := make([]byte, 32) // 32-byte dummy batch
|
||||
for i := range batch {
|
||||
batch[i] = byte(i)
|
||||
}
|
||||
|
||||
// Write until we rotate past segment 0.
|
||||
for i := 0; i < 20; i++ {
|
||||
if err := sm.AppendBatch(batch); err != nil {
|
||||
t.Fatalf("AppendBatch %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// After many writes, we should have rotated to a higher segment.
|
||||
if sm.ActiveSegmentID() == 0 {
|
||||
t.Error("expected segment rotation, but still on segment 0")
|
||||
}
|
||||
|
||||
// Verify that segment-1.wal (or higher) exists on disk.
|
||||
segment1Path := filepath.Join(dir, "segment-1.wal")
|
||||
if _, err := os.Stat(segment1Path); err != nil {
|
||||
t.Errorf("segment-1.wal should exist after rotation: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentManagerBatchNotSplit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := tinyWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
// Fill segment 0 until it's nearly full.
|
||||
smallBatch := make([]byte, 16)
|
||||
for i := range smallBatch {
|
||||
smallBatch[i] = byte(i)
|
||||
}
|
||||
|
||||
// Write until we're close to rotation threshold.
|
||||
for sm.RemainingPayload() > 256 {
|
||||
if err := sm.AppendBatch(smallBatch); err != nil {
|
||||
t.Fatalf("AppendBatch small: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Now write a batch that triggers rotation.
|
||||
// This batch must go entirely into the new segment.
|
||||
triggerBatch := make([]byte, 128)
|
||||
for i := range triggerBatch {
|
||||
triggerBatch[i] = 0xAA
|
||||
}
|
||||
|
||||
segIDBefore := sm.ActiveSegmentID()
|
||||
if err := sm.AppendBatch(triggerBatch); err != nil {
|
||||
t.Fatalf("AppendBatch trigger: %v", err)
|
||||
}
|
||||
segIDAfter := sm.ActiveSegmentID()
|
||||
|
||||
// The trigger batch should have caused rotation (or the segment was big enough).
|
||||
// If rotation happened, verify the batch is in the new segment.
|
||||
if segIDAfter != segIDBefore {
|
||||
// Rotation occurred — the batch should be in the new segment.
|
||||
// Read the new segment file and verify it contains our trigger data.
|
||||
newSegPath := filepath.Join(dir, fmtSegName(segIDAfter))
|
||||
data, err := os.ReadFile(newSegPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read new segment: %v", err)
|
||||
}
|
||||
// The trigger batch bytes should appear somewhere after the file header.
|
||||
found := false
|
||||
for i := WalFileHeaderSize; i <= len(data)-len(triggerBatch); i++ {
|
||||
if data[i] == 0xAA {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("trigger batch data not found in new segment after rotation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentManagerCurrentFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := tinyWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
// Initial CURRENT should point to segment 0.
|
||||
segID, ok := manifest.ReadCurrent(dir)
|
||||
if !ok || segID != 0 {
|
||||
t.Fatalf("initial CURRENT: got segment %d, ok=%v, want 0", segID, ok)
|
||||
}
|
||||
|
||||
// Write enough to force rotation.
|
||||
batch := make([]byte, 32)
|
||||
for i := 0; i < 20; i++ {
|
||||
if err := sm.AppendBatch(batch); err != nil {
|
||||
t.Fatalf("AppendBatch %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// CURRENT should now point to the active segment.
|
||||
currentSegID, ok := manifest.ReadCurrent(dir)
|
||||
if !ok {
|
||||
t.Fatal("ReadCurrent after rotation: expected CURRENT file to exist")
|
||||
}
|
||||
if currentSegID != sm.ActiveSegmentID() {
|
||||
t.Errorf("CURRENT segment ID = %d, want %d", currentSegID, sm.ActiveSegmentID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentManagerSync(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := testWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
if err := sm.Sync(); err != nil {
|
||||
t.Errorf("Sync: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentManagerRemainingPayload(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := testWalConfig()
|
||||
|
||||
sm, err := NewSegmentManager(dir, 0, 1, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSegmentManager: %v", err)
|
||||
}
|
||||
defer sm.Close()
|
||||
|
||||
expected := cfg.MaxSegmentSize - WalFileHeaderSize
|
||||
if got := sm.RemainingPayload(); got != expected {
|
||||
t.Errorf("RemainingPayload = %d, want %d", got, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// fmtSegName formats a segment filename.
|
||||
func fmtSegName(segID uint64) string {
|
||||
return filepath.Join("", "segment-"+itoa(segID)+".wal")
|
||||
}
|
||||
|
||||
func itoa(n uint64) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
Reference in New Issue
Block a user