feat: initialize project structure and error types
- go.mod with github.com/dailz/go-kv, Go 1.26.3, testify - config/config.go with WalConfig, Validate() with checked arithmetic - errors.go with sentinel errors (ErrCommitUnknown, ErrWriteStopped, etc.) - wal/constants.go with all WAL format constants and enums - wal/header.go with WAL File Header encode/decode (CRC32 IEEE) - wal/record.go with Physical Record codec, block boundary, SplitIntoRecords - wal/entry.go with WAL Entry codec (varint keys/values, OpType, ValueKind) - wal/sequence.go with SequenceManager (atomic, CAS, overflow-safe) - manifest/manifest.go with MANIFEST stub (Load/Save atomic) - manifest/current.go with CURRENT file (WriteCurrent/ReadCurrent) - Comprehensive tests for all modules - .golangci.yml configuration
This commit is contained in:
dailz
@@ -0,0 +1,80 @@
|
||||
package wal
|
||||
|
||||
// WAL file format constants.
|
||||
|
||||
const (
|
||||
// WalMagic is the file type identifier for WAL segment files ("WALK").
|
||||
WalMagic uint32 = 0x57414C4B
|
||||
|
||||
// WalFormatVersion is the WAL file format version. First version is 1.
|
||||
WalFormatVersion uint16 = 1
|
||||
|
||||
// WalFileHeaderSize is the size of the WAL file header in bytes.
|
||||
// Fields: magic(4) + formatVersion(2) + headerSize(2) + blockSize(4) +
|
||||
// segmentID(8) + startSequence(8) + headerCRC(4) = 32.
|
||||
WalFileHeaderSize = 32
|
||||
|
||||
// WalBlockSize is the fixed size of each WAL block in bytes (32 KB).
|
||||
WalBlockSize = 32 * 1024
|
||||
|
||||
// PhysicalRecordHeaderSize is the size of a physical record header in bytes.
|
||||
// Fields: crc32c(4) + length(2) + type(1) = 7.
|
||||
PhysicalRecordHeaderSize = 7
|
||||
|
||||
// WalBatchHeaderSize is the size of a WAL batch header in bytes.
|
||||
// Fields: flags(2) + baseSequence(8) + entryCount(4) + entriesSize(4) = 18.
|
||||
WalBatchHeaderSize = 18
|
||||
|
||||
// MaxWalBatchEntryCount limits the number of entries in a single batch.
|
||||
MaxWalBatchEntryCount uint32 = 10000
|
||||
|
||||
// MaxWalBatchEntriesSize limits the total size of the entries region in bytes (4 MB).
|
||||
MaxWalBatchEntriesSize uint32 = 4 * 1024 * 1024
|
||||
|
||||
// MaxWalKeyBytes limits the size of a single key in bytes (4 KB).
|
||||
MaxWalKeyBytes uint32 = 4 * 1024
|
||||
|
||||
// MaxWalInlineValueBytes limits the size of an inline value in bytes (4 KB).
|
||||
// Values exceeding this must use ValueLogPointer.
|
||||
MaxWalInlineValueBytes uint32 = 4 * 1024
|
||||
|
||||
// MaxWalVarintBytes is the maximum encoded length of a varint field.
|
||||
MaxWalVarintBytes = 5
|
||||
|
||||
// DefaultMaxWalSegmentSize is the default maximum size of a WAL segment file (64 MB).
|
||||
DefaultMaxWalSegmentSize uint64 = 64 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Fragment types for physical records.
|
||||
const (
|
||||
// RecInvalid is an illegal fragment type used for corruption detection.
|
||||
RecInvalid uint8 = 0
|
||||
// RecFull indicates a complete WAL batch in a single physical record.
|
||||
RecFull uint8 = 1
|
||||
// RecFirst is the first fragment of a multi-record WAL batch.
|
||||
RecFirst uint8 = 2
|
||||
// RecMiddle is a middle fragment (may appear zero or more times).
|
||||
RecMiddle uint8 = 3
|
||||
// RecLast is the last fragment of a multi-record WAL batch.
|
||||
RecLast uint8 = 4
|
||||
)
|
||||
|
||||
// OpType represents the operation type of a WAL entry.
|
||||
const (
|
||||
// OpInvalid is an illegal operation type used for corruption detection.
|
||||
OpInvalid uint8 = 0
|
||||
// OpPut represents a key-value put operation.
|
||||
OpPut uint8 = 1
|
||||
// OpDelete represents a key deletion operation.
|
||||
OpDelete uint8 = 2
|
||||
)
|
||||
|
||||
// ValueKind represents how the value field is encoded in a WAL entry.
|
||||
const (
|
||||
// VKNone indicates no value (used with Delete operations).
|
||||
VKNone uint8 = 0
|
||||
// VKInline indicates the value field contains inline user bytes.
|
||||
VKInline uint8 = 1
|
||||
// VKValueLogPointer indicates the value field contains an encoded Value Log pointer.
|
||||
VKValueLogPointer uint8 = 2
|
||||
)
|
||||
@@ -0,0 +1,72 @@
|
||||
package wal
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestConstantValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
got interface{}
|
||||
expected interface{}
|
||||
}{
|
||||
{"WalBlockSize", WalBlockSize, 32 * 1024},
|
||||
{"WalFileHeaderSize", WalFileHeaderSize, 32},
|
||||
{"WalBatchHeaderSize", WalBatchHeaderSize, 18},
|
||||
{"PhysicalRecordHeaderSize", PhysicalRecordHeaderSize, 7},
|
||||
{"MaxWalBatchEntriesSize", MaxWalBatchEntriesSize, uint32(4 * 1024 * 1024)},
|
||||
{"MaxWalBatchEntryCount", MaxWalBatchEntryCount, uint32(10000)},
|
||||
{"MaxWalKeyBytes", MaxWalKeyBytes, uint32(4 * 1024)},
|
||||
{"MaxWalInlineValueBytes", MaxWalInlineValueBytes, uint32(4 * 1024)},
|
||||
{"MaxWalVarintBytes", MaxWalVarintBytes, 5},
|
||||
{"DefaultMaxWalSegmentSize", DefaultMaxWalSegmentSize, uint64(64 * 1024 * 1024)},
|
||||
{"WalMagic", WalMagic, uint32(0x57414C4B)},
|
||||
{"WalFormatVersion", WalFormatVersion, uint16(1)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if tt.got != tt.expected {
|
||||
t.Errorf("%s = %v, want %v", tt.name, tt.got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentTypes(t *testing.T) {
|
||||
if RecInvalid != uint8(0) {
|
||||
t.Errorf("RecInvalid = %d, want 0", RecInvalid)
|
||||
}
|
||||
if RecFull != uint8(1) {
|
||||
t.Errorf("RecFull = %d, want 1", RecFull)
|
||||
}
|
||||
if RecFirst != uint8(2) {
|
||||
t.Errorf("RecFirst = %d, want 2", RecFirst)
|
||||
}
|
||||
if RecMiddle != uint8(3) {
|
||||
t.Errorf("RecMiddle = %d, want 3", RecMiddle)
|
||||
}
|
||||
if RecLast != uint8(4) {
|
||||
t.Errorf("RecLast = %d, want 4", RecLast)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpTypes(t *testing.T) {
|
||||
if OpInvalid != uint8(0) {
|
||||
t.Errorf("OpInvalid = %d, want 0", OpInvalid)
|
||||
}
|
||||
if OpPut != uint8(1) {
|
||||
t.Errorf("OpPut = %d, want 1", OpPut)
|
||||
}
|
||||
if OpDelete != uint8(2) {
|
||||
t.Errorf("OpDelete = %d, want 2", OpDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValueKinds(t *testing.T) {
|
||||
if VKNone != uint8(0) {
|
||||
t.Errorf("VKNone = %d, want 0", VKNone)
|
||||
}
|
||||
if VKInline != uint8(1) {
|
||||
t.Errorf("VKInline = %d, want 1", VKInline)
|
||||
}
|
||||
if VKValueLogPointer != uint8(2) {
|
||||
t.Errorf("VKValueLogPointer = %d, want 2", VKValueLogPointer)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package wal implements the Write-Ahead Log subsystem.
|
||||
package wal
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// WalEntry represents a single WAL record.
|
||||
type WalEntry struct {
|
||||
OpType uint8
|
||||
ValueKind uint8
|
||||
Key []byte
|
||||
Value []byte
|
||||
}
|
||||
|
||||
// Validate checks that the entry fields are consistent with the design rules.
|
||||
func (e *WalEntry) Validate() error {
|
||||
keyLen := len(e.Key)
|
||||
if keyLen == 0 || keyLen > int(MaxWalKeyBytes) {
|
||||
return fmt.Errorf("wal: invalid key length %d", keyLen)
|
||||
}
|
||||
|
||||
valLen := len(e.Value)
|
||||
|
||||
switch e.OpType {
|
||||
case OpPut:
|
||||
switch e.ValueKind {
|
||||
case VKInline:
|
||||
if valLen > int(MaxWalInlineValueBytes) {
|
||||
return fmt.Errorf("wal: inline value length %d out of range [0, %d]", valLen, MaxWalInlineValueBytes)
|
||||
}
|
||||
case VKValueLogPointer:
|
||||
if valLen == 0 {
|
||||
return errors.New("wal: value log pointer requires non-empty value")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("wal: put requires valueKind Inline(1) or ValueLogPointer(2), got %d", e.ValueKind)
|
||||
}
|
||||
|
||||
case OpDelete:
|
||||
if e.ValueKind != VKNone {
|
||||
return fmt.Errorf("wal: delete requires valueKind None(0), got %d", e.ValueKind)
|
||||
}
|
||||
if valLen != 0 {
|
||||
return fmt.Errorf("wal: delete requires empty value, got length %d", valLen)
|
||||
}
|
||||
|
||||
default:
|
||||
return fmt.Errorf("wal: invalid opType %d", e.OpType)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeEntry serializes a WalEntry into a byte slice.
|
||||
func EncodeEntry(e *WalEntry) ([]byte, error) {
|
||||
if err := e.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keyLen := uint64(len(e.Key))
|
||||
valLen := uint64(len(e.Value))
|
||||
|
||||
// Size: 1 (opType) + 1 (valueKind) + varint(keyLen) + varint(valLen) + key + value
|
||||
size := 2 + MaxWalVarintBytes + MaxWalVarintBytes + len(e.Key) + len(e.Value)
|
||||
buf := make([]byte, size)
|
||||
|
||||
buf[0] = e.OpType
|
||||
buf[1] = e.ValueKind
|
||||
n := 2
|
||||
n += binary.PutUvarint(buf[n:], keyLen)
|
||||
n += binary.PutUvarint(buf[n:], valLen)
|
||||
n += copy(buf[n:], e.Key)
|
||||
n += copy(buf[n:], e.Value)
|
||||
|
||||
return buf[:n], nil
|
||||
}
|
||||
|
||||
// DecodeEntry deserializes a WalEntry from a byte slice.
|
||||
// Returns the decoded entry and the number of bytes consumed.
|
||||
func DecodeEntry(data []byte) (entry *WalEntry, consumed int, err error) {
|
||||
if len(data) < 2 {
|
||||
return nil, 0, errors.New("wal: data too short for entry header")
|
||||
}
|
||||
|
||||
opType := data[0]
|
||||
valueKind := data[1]
|
||||
r := bytes.NewReader(data[2:])
|
||||
|
||||
keyLen, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("wal: reading key length: %w", err)
|
||||
}
|
||||
valLen, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("wal: reading value length: %w", err)
|
||||
}
|
||||
|
||||
// Calculate consumed so far: 2 header bytes + bytes read from reader
|
||||
consumed = 2 + (len(data) - 2 - r.Len())
|
||||
|
||||
// Read key
|
||||
remaining := len(data) - consumed
|
||||
if uint64(remaining) < keyLen {
|
||||
return nil, 0, fmt.Errorf("wal: data truncated: need %d bytes for key, have %d", keyLen, remaining)
|
||||
}
|
||||
key := make([]byte, keyLen)
|
||||
copy(key, data[consumed:consumed+int(keyLen)])
|
||||
consumed += int(keyLen)
|
||||
|
||||
// Read value
|
||||
remaining = len(data) - consumed
|
||||
if uint64(remaining) < valLen {
|
||||
return nil, 0, fmt.Errorf("wal: data truncated: need %d bytes for value, have %d", valLen, remaining)
|
||||
}
|
||||
value := make([]byte, valLen)
|
||||
copy(value, data[consumed:consumed+int(valLen)])
|
||||
consumed += int(valLen)
|
||||
|
||||
e := &WalEntry{
|
||||
OpType: opType,
|
||||
ValueKind: valueKind,
|
||||
Key: key,
|
||||
Value: value,
|
||||
}
|
||||
|
||||
if err := e.Validate(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return e, consumed, nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEntryRoundtrip(t *testing.T) {
|
||||
maxKey := bytes.Repeat([]byte("k"), int(MaxWalKeyBytes))
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
e *WalEntry
|
||||
}{
|
||||
{"put_inline", &WalEntry{OpPut, VKInline, []byte("key1"), []byte("val1")}},
|
||||
{"put_inline_empty_value", &WalEntry{OpPut, VKInline, []byte("key2"), []byte{}}},
|
||||
{"delete", &WalEntry{OpDelete, VKNone, []byte("key3"), nil}},
|
||||
{"put_max_key", &WalEntry{OpPut, VKInline, maxKey, []byte("v")}},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
encoded, err := EncodeEntry(tc.e)
|
||||
if err != nil {
|
||||
t.Fatalf("encode: %v", err)
|
||||
}
|
||||
|
||||
got, consumed, err := DecodeEntry(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if consumed != len(encoded) {
|
||||
t.Fatalf("consumed %d != encoded len %d", consumed, len(encoded))
|
||||
}
|
||||
|
||||
if got.OpType != tc.e.OpType {
|
||||
t.Errorf("OpType: got %d, want %d", got.OpType, tc.e.OpType)
|
||||
}
|
||||
if got.ValueKind != tc.e.ValueKind {
|
||||
t.Errorf("ValueKind: got %d, want %d", got.ValueKind, tc.e.ValueKind)
|
||||
}
|
||||
if !bytes.Equal(got.Key, tc.e.Key) {
|
||||
t.Errorf("Key: got %q, want %q", got.Key, tc.e.Key)
|
||||
}
|
||||
if !bytes.Equal(got.Value, tc.e.Value) {
|
||||
t.Errorf("Value: got %q, want %q", got.Value, tc.e.Value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEntryValidation(t *testing.T) {
|
||||
bigKey := bytes.Repeat([]byte("k"), int(MaxWalKeyBytes)+1)
|
||||
bigVal := bytes.Repeat([]byte("v"), int(MaxWalInlineValueBytes)+1)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
e *WalEntry
|
||||
wantErr bool
|
||||
}{
|
||||
{"op_invalid", &WalEntry{OpInvalid, VKNone, []byte("k"), nil}, true},
|
||||
{"put_vk_none", &WalEntry{OpPut, VKNone, []byte("k"), nil}, true},
|
||||
{"delete_vk_inline", &WalEntry{OpDelete, VKInline, []byte("k"), nil}, true},
|
||||
{"key_empty", &WalEntry{OpPut, VKInline, []byte{}, []byte("v")}, true},
|
||||
{"key_too_big", &WalEntry{OpPut, VKInline, bigKey, []byte("v")}, true},
|
||||
{"put_inline_val_too_big", &WalEntry{OpPut, VKInline, []byte("k"), bigVal}, true},
|
||||
{"put_vlptr_val_empty", &WalEntry{OpPut, VKValueLogPointer, []byte("k"), []byte{}}, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := EncodeEntry(tc.e)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Errorf("EncodeEntry() error = %v, wantErr %v", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEntryDecodeTruncated(t *testing.T) {
|
||||
// Build a valid encoded entry, then truncate mid-varint.
|
||||
e := &WalEntry{OpPut, VKInline, []byte("key1"), []byte("value1")}
|
||||
full, err := EncodeEntry(e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Truncate to just 1 byte — not enough for header.
|
||||
_, _, err = DecodeEntry(full[:1])
|
||||
if err == nil {
|
||||
t.Error("expected error for 1-byte data")
|
||||
}
|
||||
|
||||
// Build data with an incomplete varint: opType + valueKind + start of varint (0xFF means more bytes follow).
|
||||
trunc := []byte{OpPut, VKInline, 0xFF}
|
||||
_, _, err = DecodeEntry(trunc)
|
||||
if err == nil {
|
||||
t.Error("expected error for truncated varint")
|
||||
}
|
||||
|
||||
// Also test: varint specifies more bytes than available.
|
||||
// Encode a large key length varint but don't provide the key bytes.
|
||||
varintBuf := make([]byte, binary.MaxVarintLen64)
|
||||
n := binary.PutUvarint(varintBuf, 1000) // keyLen = 1000
|
||||
data := []byte{OpPut, VKInline}
|
||||
data = append(data, varintBuf[:n]...)
|
||||
data = append(data, varintBuf[:n]...) // valLen varint (also 1000)
|
||||
// Don't append any key/value bytes.
|
||||
_, _, err = DecodeEntry(data)
|
||||
if err == nil {
|
||||
t.Error("expected error for missing key/value bytes")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
const (
|
||||
walFileHeaderSize = 32
|
||||
walMagic = 0x57414C4B // "WALK" in ASCII
|
||||
walFormatVersion = 1
|
||||
)
|
||||
|
||||
// WalFileHeader is the 32-byte header written at the start of every WAL segment file.
|
||||
type WalFileHeader struct {
|
||||
Magic uint32
|
||||
FormatVersion uint16
|
||||
HeaderSize uint16
|
||||
BlockSize uint32
|
||||
SegmentID uint64
|
||||
StartSequence uint64
|
||||
HeaderCRC uint32
|
||||
}
|
||||
|
||||
// EncodeWalHeader serializes h into a fixed-size 32-byte array using little-endian byte order.
|
||||
// The HeaderCRC field is computed over bytes 0–27 (everything except the CRC itself).
|
||||
func EncodeWalHeader(h *WalFileHeader) [walFileHeaderSize]byte {
|
||||
h.HeaderSize = walFileHeaderSize
|
||||
h.Magic = walMagic
|
||||
h.FormatVersion = walFormatVersion
|
||||
|
||||
var buf [walFileHeaderSize]byte
|
||||
le := binary.LittleEndian
|
||||
|
||||
le.PutUint32(buf[0:4], h.Magic)
|
||||
le.PutUint16(buf[4:6], h.FormatVersion)
|
||||
le.PutUint16(buf[6:8], h.HeaderSize)
|
||||
le.PutUint32(buf[8:12], h.BlockSize)
|
||||
le.PutUint64(buf[12:20], h.SegmentID)
|
||||
le.PutUint64(buf[20:28], h.StartSequence)
|
||||
|
||||
// CRC32 IEEE over bytes 0–27 (excludes the CRC field itself)
|
||||
h.HeaderCRC = crc32.ChecksumIEEE(buf[0:28])
|
||||
le.PutUint32(buf[28:32], h.HeaderCRC)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
var (
|
||||
errBadMagic = errors.New("wal: bad magic number")
|
||||
errBadVersion = errors.New("wal: unsupported format version")
|
||||
errBadHeaderSize = errors.New("wal: bad header size")
|
||||
errCRCMismatch = errors.New("wal: header CRC mismatch")
|
||||
errHeaderTooShort = errors.New("wal: header data too short")
|
||||
)
|
||||
|
||||
// DecodeWalHeader parses a 32-byte little-endian header and validates magic,
|
||||
// format version, header size, and CRC.
|
||||
func DecodeWalHeader(data []byte) (*WalFileHeader, error) {
|
||||
if len(data) < walFileHeaderSize {
|
||||
return nil, errHeaderTooShort
|
||||
}
|
||||
|
||||
le := binary.LittleEndian
|
||||
|
||||
magic := le.Uint32(data[0:4])
|
||||
if magic != walMagic {
|
||||
return nil, errBadMagic
|
||||
}
|
||||
|
||||
version := le.Uint16(data[4:6])
|
||||
if version != walFormatVersion {
|
||||
return nil, errBadVersion
|
||||
}
|
||||
|
||||
hdrSize := le.Uint16(data[6:8])
|
||||
if hdrSize != walFileHeaderSize {
|
||||
return nil, errBadHeaderSize
|
||||
}
|
||||
|
||||
// Verify CRC before trusting any other fields
|
||||
gotCRC := crc32.ChecksumIEEE(data[0:28])
|
||||
storedCRC := le.Uint32(data[28:32])
|
||||
if gotCRC != storedCRC {
|
||||
return nil, errCRCMismatch
|
||||
}
|
||||
|
||||
return &WalFileHeader{
|
||||
Magic: magic,
|
||||
FormatVersion: version,
|
||||
HeaderSize: hdrSize,
|
||||
BlockSize: le.Uint32(data[8:12]),
|
||||
SegmentID: le.Uint64(data[12:20]),
|
||||
StartSequence: le.Uint64(data[20:28]),
|
||||
HeaderCRC: storedCRC,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHeaderRoundtrip(t *testing.T) {
|
||||
orig := &WalFileHeader{
|
||||
BlockSize: 32 * 1024, // 32 KB
|
||||
SegmentID: 5,
|
||||
StartSequence: 1000,
|
||||
}
|
||||
|
||||
encoded := EncodeWalHeader(orig)
|
||||
decoded, err := DecodeWalHeader(encoded[:])
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeWalHeader returned error: %v", err)
|
||||
}
|
||||
|
||||
if decoded.Magic != walMagic {
|
||||
t.Errorf("Magic = %x, want %x", decoded.Magic, walMagic)
|
||||
}
|
||||
if decoded.FormatVersion != walFormatVersion {
|
||||
t.Errorf("FormatVersion = %d, want %d", decoded.FormatVersion, walFormatVersion)
|
||||
}
|
||||
if decoded.HeaderSize != walFileHeaderSize {
|
||||
t.Errorf("HeaderSize = %d, want %d", decoded.HeaderSize, walFileHeaderSize)
|
||||
}
|
||||
if decoded.BlockSize != orig.BlockSize {
|
||||
t.Errorf("BlockSize = %d, want %d", decoded.BlockSize, orig.BlockSize)
|
||||
}
|
||||
if decoded.SegmentID != orig.SegmentID {
|
||||
t.Errorf("SegmentID = %d, want %d", decoded.SegmentID, orig.SegmentID)
|
||||
}
|
||||
if decoded.StartSequence != orig.StartSequence {
|
||||
t.Errorf("StartSequence = %d, want %d", decoded.StartSequence, orig.StartSequence)
|
||||
}
|
||||
if decoded.HeaderCRC != orig.HeaderCRC {
|
||||
t.Errorf("HeaderCRC = %x, want %x", decoded.HeaderCRC, orig.HeaderCRC)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderCRC(t *testing.T) {
|
||||
h := &WalFileHeader{
|
||||
BlockSize: 32 * 1024,
|
||||
SegmentID: 1,
|
||||
StartSequence: 0,
|
||||
}
|
||||
encoded := EncodeWalHeader(h)
|
||||
|
||||
// Flip a byte in the magic field (bytes 0-3)
|
||||
encoded[0] ^= 0xFF
|
||||
|
||||
_, err := DecodeWalHeader(encoded[:])
|
||||
if !errors.Is(err, errCRCMismatch) && !errors.Is(err, errBadMagic) {
|
||||
// Flipping magic may fail on magic check first or CRC check
|
||||
// Either way, decoding must fail
|
||||
t.Fatalf("expected CRC or magic error, got: %v", err)
|
||||
}
|
||||
|
||||
// Restore magic and flip a byte in the payload instead
|
||||
encoded[0] = byte(walMagic & 0xFF)
|
||||
encoded[12] ^= 0x01 // flip byte in SegmentID
|
||||
|
||||
_, err = DecodeWalHeader(encoded[:])
|
||||
if !errors.Is(err, errCRCMismatch) {
|
||||
t.Fatalf("expected errCRCMismatch, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderBadMagic(t *testing.T) {
|
||||
data := make([]byte, 32)
|
||||
// All zeros — magic won't match
|
||||
_, err := DecodeWalHeader(data)
|
||||
if !errors.Is(err, errBadMagic) {
|
||||
t.Fatalf("expected errBadMagic, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderShortData(t *testing.T) {
|
||||
data := make([]byte, 16) // too short
|
||||
_, err := DecodeWalHeader(data)
|
||||
if !errors.Is(err, errHeaderTooShort) {
|
||||
t.Fatalf("expected errHeaderTooShort, got: %v", err)
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
// PhysicalRecord represents a single physical record in the WAL.
|
||||
type PhysicalRecord struct {
|
||||
CRC uint32
|
||||
Length uint16
|
||||
Type uint8
|
||||
Payload []byte
|
||||
}
|
||||
|
||||
// EncodePhysicalRecord encodes a physical record with the given type and payload.
|
||||
// Format: [crc32 u32 LE][length u16 LE][type u8][payload bytes]
|
||||
// CRC covers length + type + payload.
|
||||
func EncodePhysicalRecord(recType uint8, payload []byte) []byte {
|
||||
length := uint16(len(payload))
|
||||
buf := make([]byte, PhysicalRecordHeaderSize+len(payload))
|
||||
|
||||
// Write length and type first so we can compute CRC.
|
||||
binary.LittleEndian.PutUint16(buf[4:6], length)
|
||||
buf[6] = recType
|
||||
copy(buf[7:], payload)
|
||||
|
||||
// CRC covers bytes [4:] = length + type + payload.
|
||||
crc := crc32.ChecksumIEEE(buf[4:])
|
||||
binary.LittleEndian.PutUint32(buf[0:4], crc)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
// DecodePhysicalRecord decodes a physical record from data.
|
||||
// Returns the record, number of bytes consumed, and any error.
|
||||
func DecodePhysicalRecord(data []byte) (rec *PhysicalRecord, consumed int, err error) {
|
||||
if len(data) < PhysicalRecordHeaderSize {
|
||||
return nil, 0, errors.New("record: data too short for header")
|
||||
}
|
||||
|
||||
crc := binary.LittleEndian.Uint32(data[0:4])
|
||||
length := binary.LittleEndian.Uint16(data[4:6])
|
||||
recType := data[6]
|
||||
|
||||
if int(length) > len(data)-PhysicalRecordHeaderSize {
|
||||
return nil, 0, errors.New("record: data too short for payload")
|
||||
}
|
||||
|
||||
payload := make([]byte, length)
|
||||
copy(payload, data[7:7+length])
|
||||
|
||||
// Verify CRC: covers length + type + payload.
|
||||
expectedCRC := crc32.ChecksumIEEE(data[4 : 7+length])
|
||||
if crc != expectedCRC {
|
||||
return nil, 0, errors.New("record: CRC mismatch")
|
||||
}
|
||||
|
||||
consumed = PhysicalRecordHeaderSize + int(length)
|
||||
return &PhysicalRecord{
|
||||
CRC: crc,
|
||||
Length: length,
|
||||
Type: recType,
|
||||
Payload: payload,
|
||||
}, consumed, nil
|
||||
}
|
||||
|
||||
// PaddingNeeded returns the number of padding bytes needed at blockOffset.
|
||||
// If the remaining space in the current block is <= PhysicalRecordHeaderSize (7),
|
||||
// that remaining space must be zero-padded.
|
||||
func PaddingNeeded(blockOffset uint32) int {
|
||||
remaining := WalBlockSize - (blockOffset % WalBlockSize)
|
||||
if remaining <= PhysicalRecordHeaderSize {
|
||||
return int(remaining)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// CanFitRecord reports whether a physical record with the given payload length
|
||||
// can fit in the current block starting at blockOffset.
|
||||
func CanFitRecord(blockOffset uint32, payloadLen uint32) bool {
|
||||
remaining := WalBlockSize - (blockOffset % WalBlockSize)
|
||||
return int(remaining) >= PhysicalRecordHeaderSize+int(payloadLen)
|
||||
}
|
||||
|
||||
// SplitIntoRecords splits an encoded WAL batch into physical record payloads
|
||||
// respecting 32 KB block boundaries.
|
||||
// Each returned byte slice is the full encoded physical record (header + payload).
|
||||
func SplitIntoRecords(encodedBatch []byte) [][]byte {
|
||||
maxPayload := WalBlockSize - PhysicalRecordHeaderSize
|
||||
total := len(encodedBatch)
|
||||
|
||||
if total == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Single record fits entirely.
|
||||
if total <= maxPayload {
|
||||
return [][]byte{EncodePhysicalRecord(RecFull, encodedBatch)}
|
||||
}
|
||||
|
||||
var records [][]byte
|
||||
offset := 0
|
||||
|
||||
for offset < total {
|
||||
chunkLen := min(total-offset, maxPayload)
|
||||
|
||||
var recType uint8
|
||||
switch {
|
||||
case offset == 0 && offset+chunkLen == total:
|
||||
recType = RecFull
|
||||
case offset == 0:
|
||||
recType = RecFirst
|
||||
case offset+chunkLen == total:
|
||||
recType = RecLast
|
||||
default:
|
||||
recType = RecMiddle
|
||||
}
|
||||
|
||||
records = append(records, EncodePhysicalRecord(recType, encodedBatch[offset:offset+chunkLen]))
|
||||
offset += chunkLen
|
||||
}
|
||||
|
||||
return records
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRecordRoundtrip(t *testing.T) {
|
||||
payload := []byte("hello world")
|
||||
encoded := EncodePhysicalRecord(RecFull, payload)
|
||||
|
||||
rec, consumed, err := DecodePhysicalRecord(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePhysicalRecord failed: %v", err)
|
||||
}
|
||||
|
||||
if rec.Type != RecFull {
|
||||
t.Errorf("expected type RecFull(%d), got %d", RecFull, rec.Type)
|
||||
}
|
||||
if string(rec.Payload) != "hello world" {
|
||||
t.Errorf("expected payload 'hello world', got %q", string(rec.Payload))
|
||||
}
|
||||
if consumed != 7+len(payload) {
|
||||
t.Errorf("expected consumed %d, got %d", 7+len(payload), consumed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSmallPayload(t *testing.T) {
|
||||
payload := make([]byte, 100)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i)
|
||||
}
|
||||
|
||||
records := SplitIntoRecords(payload)
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(records))
|
||||
}
|
||||
|
||||
rec, _, err := DecodePhysicalRecord(records[0])
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePhysicalRecord failed: %v", err)
|
||||
}
|
||||
if rec.Type != RecFull {
|
||||
t.Errorf("expected RecFull, got %d", rec.Type)
|
||||
}
|
||||
if !bytes.Equal(rec.Payload, payload) {
|
||||
t.Error("payload mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitIntoRecords(t *testing.T) {
|
||||
// 40 KB payload → needs to split across blocks.
|
||||
payload := make([]byte, 40*1024)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
records := SplitIntoRecords(payload)
|
||||
if len(records) < 2 {
|
||||
t.Fatalf("expected at least 2 records, got %d", len(records))
|
||||
}
|
||||
|
||||
// Verify fragment sequence.
|
||||
types := make([]uint8, len(records))
|
||||
var concatenated []byte
|
||||
for i, enc := range records {
|
||||
rec, _, err := DecodePhysicalRecord(enc)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePhysicalRecord record %d failed: %v", i, err)
|
||||
}
|
||||
types[i] = rec.Type
|
||||
concatenated = append(concatenated, rec.Payload...)
|
||||
}
|
||||
|
||||
// First record must be RecFirst.
|
||||
if types[0] != RecFirst {
|
||||
t.Errorf("first record type: expected RecFirst(%d), got %d", RecFirst, types[0])
|
||||
}
|
||||
// Last record must be RecLast.
|
||||
if types[len(types)-1] != RecLast {
|
||||
t.Errorf("last record type: expected RecLast(%d), got %d", RecLast, types[len(types)-1])
|
||||
}
|
||||
// Middle records must be RecMiddle.
|
||||
for i := 1; i < len(types)-1; i++ {
|
||||
if types[i] != RecMiddle {
|
||||
t.Errorf("record %d type: expected RecMiddle(%d), got %d", i, RecMiddle, types[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Concatenated payloads must equal original.
|
||||
if !bytes.Equal(concatenated, payload) {
|
||||
t.Error("concatenated payloads do not match original")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockPadding(t *testing.T) {
|
||||
// blockOffset = WalBlockSize - 5 → remaining = 5, which is <= 7 → padding needed = 5.
|
||||
blockOffset := uint32(WalBlockSize - 5)
|
||||
padding := PaddingNeeded(blockOffset)
|
||||
if padding != 5 {
|
||||
t.Errorf("PaddingNeeded(%d): expected 5, got %d", blockOffset, padding)
|
||||
}
|
||||
|
||||
// Cannot fit a record.
|
||||
if CanFitRecord(blockOffset, 1) {
|
||||
t.Error("CanFitRecord should return false when remaining <= 7")
|
||||
}
|
||||
|
||||
// blockOffset = WalBlockSize - 8 → remaining = 8, which is > 7 → no padding needed.
|
||||
blockOffset2 := uint32(WalBlockSize - 8)
|
||||
padding2 := PaddingNeeded(blockOffset2)
|
||||
if padding2 != 0 {
|
||||
t.Errorf("PaddingNeeded(%d): expected 0, got %d", blockOffset2, padding2)
|
||||
}
|
||||
|
||||
// Can fit a 1-byte payload: remaining=8, header=7, payload=1 → 8 >= 8.
|
||||
if !CanFitRecord(blockOffset2, 1) {
|
||||
t.Error("CanFitRecord should return true when remaining=8 and payloadLen=1")
|
||||
}
|
||||
|
||||
// Cannot fit a 2-byte payload: remaining=8, header=7, payload=2 → 8 < 9.
|
||||
if CanFitRecord(blockOffset2, 2) {
|
||||
t.Error("CanFitRecord should return false when remaining=8 and payloadLen=2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRCMismatch(t *testing.T) {
|
||||
encoded := EncodePhysicalRecord(RecFull, []byte("test"))
|
||||
// Corrupt a payload byte.
|
||||
encoded[8] ^= 0xFF
|
||||
|
||||
_, _, err := DecodePhysicalRecord(encoded)
|
||||
if err == nil {
|
||||
t.Error("expected CRC mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataTooShort(t *testing.T) {
|
||||
_, _, err := DecodePhysicalRecord([]byte{1, 2, 3})
|
||||
if err == nil {
|
||||
t.Error("expected error for data too short")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/dailz/go-kv"
|
||||
)
|
||||
|
||||
// SequenceManager manages monotonic sequence number allocation for the WAL.
|
||||
// Invariant: durableSequence <= publishedSequence <= nextSequence
|
||||
type SequenceManager struct {
|
||||
nextSequence atomic.Uint64
|
||||
publishedSequence atomic.Uint64
|
||||
durableSequence atomic.Uint64
|
||||
exhausted atomic.Bool
|
||||
}
|
||||
|
||||
// NewSequenceManager creates a SequenceManager initialised from a recovered
|
||||
// sequence number. All three watermarks start at recoveredSequence.
|
||||
func NewSequenceManager(recoveredSequence uint64) *SequenceManager {
|
||||
sm := &SequenceManager{}
|
||||
sm.nextSequence.Store(recoveredSequence)
|
||||
sm.publishedSequence.Store(recoveredSequence)
|
||||
sm.durableSequence.Store(recoveredSequence)
|
||||
return sm
|
||||
}
|
||||
|
||||
// AllocateBatch atomically reserves [base, base+count-1] sequence numbers.
|
||||
// Returns ErrSequenceExhausted if count is 0 or the allocation would overflow uint64.
|
||||
func (sm *SequenceManager) AllocateBatch(count uint32) (baseSequence uint64, err error) {
|
||||
if count == 0 {
|
||||
return 0, go_kv.ErrSequenceExhausted
|
||||
}
|
||||
for {
|
||||
if sm.exhausted.Load() {
|
||||
return 0, go_kv.ErrSequenceExhausted
|
||||
}
|
||||
base := sm.nextSequence.Load()
|
||||
last := base + uint64(count) - 1
|
||||
if last < base {
|
||||
return 0, go_kv.ErrSequenceExhausted
|
||||
}
|
||||
newNext := last + 1
|
||||
if !sm.nextSequence.CompareAndSwap(base, newNext) {
|
||||
continue
|
||||
}
|
||||
if newNext == 0 {
|
||||
sm.exhausted.Store(true)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Publish advances the publishedSequence watermark to seq (only forward).
|
||||
func (sm *SequenceManager) Publish(seq uint64) {
|
||||
for {
|
||||
current := sm.publishedSequence.Load()
|
||||
if seq <= current {
|
||||
return
|
||||
}
|
||||
if sm.publishedSequence.CompareAndSwap(current, seq) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MarkDurable advances the durableSequence watermark to seq (only forward).
|
||||
func (sm *SequenceManager) MarkDurable(seq uint64) {
|
||||
for {
|
||||
current := sm.durableSequence.Load()
|
||||
if seq <= current {
|
||||
return
|
||||
}
|
||||
if sm.durableSequence.CompareAndSwap(current, seq) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Published returns the current publishedSequence watermark.
|
||||
func (sm *SequenceManager) Published() uint64 { return sm.publishedSequence.Load() }
|
||||
|
||||
// Durable returns the current durableSequence watermark.
|
||||
func (sm *SequenceManager) Durable() uint64 { return sm.durableSequence.Load() }
|
||||
|
||||
// NextSequence returns the next sequence number to be allocated.
|
||||
func (sm *SequenceManager) NextSequence() uint64 { return sm.nextSequence.Load() }
|
||||
@@ -0,0 +1,154 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/dailz/go-kv"
|
||||
)
|
||||
|
||||
func TestSequenceAllocation(t *testing.T) {
|
||||
sm := NewSequenceManager(0)
|
||||
|
||||
base, err := sm.AllocateBatch(5)
|
||||
if err != nil {
|
||||
t.Fatalf("AllocateBatch(5): %v", err)
|
||||
}
|
||||
if base != 0 {
|
||||
t.Fatalf("expected base=0, got %d", base)
|
||||
}
|
||||
|
||||
base, err = sm.AllocateBatch(3)
|
||||
if err != nil {
|
||||
t.Fatalf("AllocateBatch(3): %v", err)
|
||||
}
|
||||
if base != 5 {
|
||||
t.Fatalf("expected base=5, got %d", base)
|
||||
}
|
||||
|
||||
base, err = sm.AllocateBatch(1)
|
||||
if err != nil {
|
||||
t.Fatalf("AllocateBatch(1): %v", err)
|
||||
}
|
||||
if base != 8 {
|
||||
t.Fatalf("expected base=8, got %d", base)
|
||||
}
|
||||
|
||||
if sm.NextSequence() != 9 {
|
||||
t.Fatalf("expected NextSequence=9, got %d", sm.NextSequence())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequenceOverflow(t *testing.T) {
|
||||
nearMax := uint64(math.MaxUint64 - 2)
|
||||
sm := NewSequenceManager(nearMax)
|
||||
|
||||
// Remaining: MaxUint64-2, MaxUint64-1, MaxUint64 = 3 slots.
|
||||
// Asking for 5 should overflow.
|
||||
_, err := sm.AllocateBatch(5)
|
||||
if !errors.Is(err, go_kv.ErrSequenceExhausted) {
|
||||
t.Fatalf("expected ErrSequenceExhausted, got %v", err)
|
||||
}
|
||||
|
||||
// 3 should still succeed.
|
||||
base, err := sm.AllocateBatch(3)
|
||||
if err != nil {
|
||||
t.Fatalf("AllocateBatch(3): %v", err)
|
||||
}
|
||||
if base != nearMax {
|
||||
t.Fatalf("expected base=%d, got %d", nearMax, base)
|
||||
}
|
||||
|
||||
// Now any further allocation should fail.
|
||||
_, err = sm.AllocateBatch(1)
|
||||
if !errors.Is(err, go_kv.ErrSequenceExhausted) {
|
||||
t.Fatalf("expected ErrSequenceExhausted after exhaustion, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishAdvance(t *testing.T) {
|
||||
sm := NewSequenceManager(0)
|
||||
|
||||
sm.Publish(10)
|
||||
if sm.Published() != 10 {
|
||||
t.Fatalf("expected Published=10, got %d", sm.Published())
|
||||
}
|
||||
|
||||
// Publishing a lower value must not decrease the watermark.
|
||||
sm.Publish(5)
|
||||
if sm.Published() != 10 {
|
||||
t.Fatalf("expected Published=10 (no decrease), got %d", sm.Published())
|
||||
}
|
||||
|
||||
sm.Publish(15)
|
||||
if sm.Published() != 15 {
|
||||
t.Fatalf("expected Published=15, got %d", sm.Published())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkDurable(t *testing.T) {
|
||||
sm := NewSequenceManager(0)
|
||||
|
||||
sm.MarkDurable(8)
|
||||
if sm.Durable() != 8 {
|
||||
t.Fatalf("expected Durable=8, got %d", sm.Durable())
|
||||
}
|
||||
|
||||
// Lower value must not decrease.
|
||||
sm.MarkDurable(3)
|
||||
if sm.Durable() != 8 {
|
||||
t.Fatalf("expected Durable=8 (no decrease), got %d", sm.Durable())
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroCountRejected(t *testing.T) {
|
||||
sm := NewSequenceManager(0)
|
||||
|
||||
_, err := sm.AllocateBatch(0)
|
||||
if !errors.Is(err, go_kv.ErrSequenceExhausted) {
|
||||
t.Fatalf("expected ErrSequenceExhausted for count=0, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentAllocation(t *testing.T) {
|
||||
const goroutines = 16
|
||||
const batchSize uint32 = 100
|
||||
|
||||
sm := NewSequenceManager(0)
|
||||
|
||||
var totalAllocated atomic.Uint64
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 50; j++ {
|
||||
base, err := sm.AllocateBatch(batchSize)
|
||||
if err != nil {
|
||||
t.Errorf("AllocateBatch failed: %v", err)
|
||||
return
|
||||
}
|
||||
totalAllocated.Add(uint64(batchSize))
|
||||
|
||||
// Verify no overlap: base must be aligned to batchSize increments
|
||||
// and within valid range. The key property is no gaps.
|
||||
_ = base
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
expected := uint64(goroutines) * 50 * uint64(batchSize)
|
||||
if totalAllocated.Load() != expected {
|
||||
t.Fatalf("expected total allocated=%d, got %d", expected, totalAllocated.Load())
|
||||
}
|
||||
|
||||
if sm.NextSequence() != expected {
|
||||
t.Fatalf("expected NextSequence=%d, got %d", expected, sm.NextSequence())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user