feat(wal): implement batch codec, validation, and Arena allocator
- wal/batch.go: WalBatch encode/decode with FragmentCollector state machine - wal/validate.go: ValidateBatchLimits with checked arithmetic - memtable/arena.go: Arena allocator with 8-byte alignment and mutex - Comprehensive tests for all modules, all pass with -race
This commit is contained in:
dailz
+216
@@ -0,0 +1,216 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// WalBatch represents a single WAL batch — the physical persistence unit.
|
||||
// A batch consists of an 18-byte header followed by a contiguous Entries region.
|
||||
type WalBatch struct {
|
||||
Flags uint16
|
||||
BaseSequence uint64
|
||||
EntryCount uint32
|
||||
EntriesSize uint32
|
||||
Entries []byte
|
||||
}
|
||||
|
||||
// EncodeWalBatch encodes a WAL batch from the given base sequence and entries.
|
||||
// The returned byte slice is: BatchHeader(18) + encoded entries.
|
||||
func EncodeWalBatch(baseSequence uint64, entries []*WalEntry) ([]byte, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, errors.New("wal: batch requires at least one entry")
|
||||
}
|
||||
if uint32(len(entries)) > MaxWalBatchEntryCount {
|
||||
return nil, fmt.Errorf("wal: entry count %d exceeds maximum %d", len(entries), MaxWalBatchEntryCount)
|
||||
}
|
||||
|
||||
// Encode all entries.
|
||||
var entriesBuf bytes.Buffer
|
||||
for i, e := range entries {
|
||||
encoded, err := EncodeEntry(e)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wal: encoding entry %d: %w", i, err)
|
||||
}
|
||||
entriesBuf.Write(encoded)
|
||||
}
|
||||
|
||||
entriesSize := uint32(entriesBuf.Len())
|
||||
if entriesSize > MaxWalBatchEntriesSize {
|
||||
return nil, fmt.Errorf("wal: entries size %d exceeds maximum %d", entriesSize, MaxWalBatchEntriesSize)
|
||||
}
|
||||
|
||||
// Build batch: header(18) + entries.
|
||||
buf := make([]byte, WalBatchHeaderSize+entriesSize)
|
||||
binary.LittleEndian.PutUint16(buf[0:2], 0) // flags
|
||||
binary.LittleEndian.PutUint64(buf[2:10], baseSequence)
|
||||
binary.LittleEndian.PutUint32(buf[10:14], uint32(len(entries)))
|
||||
binary.LittleEndian.PutUint32(buf[14:18], entriesSize)
|
||||
copy(buf[WalBatchHeaderSize:], entriesBuf.Bytes())
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// DecodeWalBatch decodes a WAL batch from raw bytes.
|
||||
// The data must contain the full batch (header + entries).
|
||||
func DecodeWalBatch(data []byte) (*WalBatch, error) {
|
||||
if len(data) < WalBatchHeaderSize {
|
||||
return nil, fmt.Errorf("wal: batch data too short: %d < %d", len(data), WalBatchHeaderSize)
|
||||
}
|
||||
|
||||
flags := binary.LittleEndian.Uint16(data[0:2])
|
||||
baseSeq := binary.LittleEndian.Uint64(data[2:10])
|
||||
entryCount := binary.LittleEndian.Uint32(data[10:14])
|
||||
entriesSize := binary.LittleEndian.Uint32(data[14:18])
|
||||
|
||||
if entryCount == 0 {
|
||||
return nil, errors.New("wal: batch entry count is zero")
|
||||
}
|
||||
if entryCount > MaxWalBatchEntryCount {
|
||||
return nil, fmt.Errorf("wal: entry count %d exceeds maximum %d", entryCount, MaxWalBatchEntryCount)
|
||||
}
|
||||
if entriesSize > MaxWalBatchEntriesSize {
|
||||
return nil, fmt.Errorf("wal: entries size %d exceeds maximum %d", entriesSize, MaxWalBatchEntriesSize)
|
||||
}
|
||||
|
||||
expectedLen := WalBatchHeaderSize + entriesSize
|
||||
if uint32(len(data)) < expectedLen {
|
||||
return nil, fmt.Errorf("wal: entries size mismatch: header says %d bytes, have %d bytes after header",
|
||||
entriesSize, len(data)-WalBatchHeaderSize)
|
||||
}
|
||||
|
||||
entries := make([]byte, entriesSize)
|
||||
copy(entries, data[WalBatchHeaderSize:WalBatchHeaderSize+entriesSize])
|
||||
|
||||
return &WalBatch{
|
||||
Flags: flags,
|
||||
BaseSequence: baseSeq,
|
||||
EntryCount: entryCount,
|
||||
EntriesSize: entriesSize,
|
||||
Entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FragmentState represents the state of the fragment collector state machine.
|
||||
type FragmentState uint8
|
||||
|
||||
const (
|
||||
// FragmentIdle means no fragments are being collected.
|
||||
FragmentIdle FragmentState = 0
|
||||
// FragmentCollecting means fragments are being accumulated.
|
||||
FragmentCollecting FragmentState = 1
|
||||
)
|
||||
|
||||
// maxFragmentBufferSize is the maximum total bytes the fragment buffer can hold:
|
||||
// Batch Header size + max entries size.
|
||||
var maxFragmentBufferSize = uint32(WalBatchHeaderSize) + MaxWalBatchEntriesSize
|
||||
|
||||
// FragmentCollector reassembles WAL batches from physical record fragments.
|
||||
// The state machine transitions between Idle and Collecting based on the
|
||||
// record type (Full, First, Middle, Last).
|
||||
type FragmentCollector struct {
|
||||
state FragmentState
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
// NewFragmentCollector creates a new FragmentCollector in Idle state.
|
||||
func NewFragmentCollector() *FragmentCollector {
|
||||
return &FragmentCollector{
|
||||
state: FragmentIdle,
|
||||
}
|
||||
}
|
||||
|
||||
// Reset clears all collected data and returns the collector to Idle state.
|
||||
func (fc *FragmentCollector) Reset() {
|
||||
fc.state = FragmentIdle
|
||||
fc.buf.Reset()
|
||||
}
|
||||
|
||||
// Append feeds a physical record fragment to the collector.
|
||||
// The state machine enforces valid transitions:
|
||||
// - Idle + Full → collect payload, stay Idle (complete)
|
||||
// - Idle + First → collect payload, → Collecting
|
||||
// - Idle + Middle → error
|
||||
// - Idle + Last → error
|
||||
// - Collecting + Middle → collect payload
|
||||
// - Collecting + Last → collect payload, → Idle (complete)
|
||||
// - Collecting + Full → error
|
||||
// - Collecting + First → error
|
||||
func (fc *FragmentCollector) Append(recType uint8, payload []byte) error {
|
||||
switch fc.state {
|
||||
case FragmentIdle:
|
||||
switch recType {
|
||||
case RecFull:
|
||||
// Complete batch in one record.
|
||||
if err := fc.checkBufferCapacity(len(payload)); err != nil {
|
||||
return err
|
||||
}
|
||||
fc.buf.Write(payload)
|
||||
// State stays Idle — batch is complete.
|
||||
return nil
|
||||
case RecFirst:
|
||||
// Start collecting fragments.
|
||||
if err := fc.checkBufferCapacity(len(payload)); err != nil {
|
||||
return err
|
||||
}
|
||||
fc.buf.Write(payload)
|
||||
fc.state = FragmentCollecting
|
||||
return nil
|
||||
case RecMiddle, RecLast:
|
||||
return fmt.Errorf("wal: unexpected fragment type %d in Idle state", recType)
|
||||
default:
|
||||
return fmt.Errorf("wal: invalid fragment type %d", recType)
|
||||
}
|
||||
|
||||
case FragmentCollecting:
|
||||
switch recType {
|
||||
case RecMiddle:
|
||||
if err := fc.checkBufferCapacity(len(payload)); err != nil {
|
||||
return err
|
||||
}
|
||||
fc.buf.Write(payload)
|
||||
return nil
|
||||
case RecLast:
|
||||
if err := fc.checkBufferCapacity(len(payload)); err != nil {
|
||||
return err
|
||||
}
|
||||
fc.buf.Write(payload)
|
||||
fc.state = FragmentIdle
|
||||
return nil
|
||||
case RecFull, RecFirst:
|
||||
return fmt.Errorf("wal: unexpected fragment type %d in Collecting state", recType)
|
||||
default:
|
||||
return fmt.Errorf("wal: invalid fragment type %d", recType)
|
||||
}
|
||||
|
||||
default:
|
||||
return fmt.Errorf("wal: invalid fragment collector state %d", fc.state)
|
||||
}
|
||||
}
|
||||
|
||||
// IsComplete reports whether a complete batch has been collected.
|
||||
// A batch is complete when the state returns to Idle after collecting data.
|
||||
func (fc *FragmentCollector) IsComplete() bool {
|
||||
return fc.state == FragmentIdle && fc.buf.Len() > 0
|
||||
}
|
||||
|
||||
// BatchData returns the collected batch bytes. Only valid when IsComplete() is true.
|
||||
func (fc *FragmentCollector) BatchData() []byte {
|
||||
return fc.buf.Bytes()
|
||||
}
|
||||
|
||||
// State returns the current fragment collector state.
|
||||
func (fc *FragmentCollector) State() FragmentState {
|
||||
return fc.state
|
||||
}
|
||||
|
||||
// checkBufferCapacity ensures the total collected bytes do not exceed the limit.
|
||||
func (fc *FragmentCollector) checkBufferCapacity(additional int) error {
|
||||
newSize := uint32(fc.buf.Len() + additional)
|
||||
if newSize > maxFragmentBufferSize {
|
||||
return fmt.Errorf("wal: fragment buffer size %d exceeds maximum %d", newSize, maxFragmentBufferSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// helper: make a Put+Inline entry.
|
||||
func makePutEntry(key, value string) *WalEntry {
|
||||
return &WalEntry{
|
||||
OpType: OpPut,
|
||||
ValueKind: VKInline,
|
||||
Key: []byte(key),
|
||||
Value: []byte(value),
|
||||
}
|
||||
}
|
||||
|
||||
// helper: make a Delete entry.
|
||||
func makeDeleteEntry(key string) *WalEntry {
|
||||
return &WalEntry{
|
||||
OpType: OpDelete,
|
||||
ValueKind: VKNone,
|
||||
Key: []byte(key),
|
||||
Value: nil,
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRoundtrip(t *testing.T) {
|
||||
// Build 10 entries: 5 Put+Inline, 3 Delete, 2 Put+Inline with empty value.
|
||||
entries := []*WalEntry{
|
||||
makePutEntry("key1", "val1"),
|
||||
makePutEntry("key2", "val2"),
|
||||
makePutEntry("key3", "val3"),
|
||||
makePutEntry("key4", "val4"),
|
||||
makePutEntry("key5", "val5"),
|
||||
makeDeleteEntry("key6"),
|
||||
makeDeleteEntry("key7"),
|
||||
makeDeleteEntry("key8"),
|
||||
makePutEntry("key9", ""), // empty value
|
||||
makePutEntry("key10", ""), // empty value
|
||||
}
|
||||
|
||||
const baseSeq uint64 = 42
|
||||
|
||||
// Encode.
|
||||
encoded, err := EncodeWalBatch(baseSeq, entries)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeWalBatch: %v", err)
|
||||
}
|
||||
|
||||
// Decode.
|
||||
batch, err := DecodeWalBatch(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeWalBatch: %v", err)
|
||||
}
|
||||
|
||||
// Verify header fields.
|
||||
if batch.Flags != 0 {
|
||||
t.Errorf("Flags = %d, want 0", batch.Flags)
|
||||
}
|
||||
if batch.BaseSequence != baseSeq {
|
||||
t.Errorf("BaseSequence = %d, want %d", batch.BaseSequence, baseSeq)
|
||||
}
|
||||
if batch.EntryCount != 10 {
|
||||
t.Errorf("EntryCount = %d, want 10", batch.EntryCount)
|
||||
}
|
||||
if batch.EntriesSize != uint32(len(batch.Entries)) {
|
||||
t.Errorf("EntriesSize = %d, len(Entries) = %d", batch.EntriesSize, len(batch.Entries))
|
||||
}
|
||||
if uint32(len(encoded)) != WalBatchHeaderSize+batch.EntriesSize {
|
||||
t.Errorf("total encoded = %d, want %d", len(encoded), WalBatchHeaderSize+batch.EntriesSize)
|
||||
}
|
||||
|
||||
// Parse entries from Entries bytes.
|
||||
offset := 0
|
||||
for i, want := range entries {
|
||||
got, consumed, err := DecodeEntry(batch.Entries[offset:])
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeEntry[%d]: %v", i, err)
|
||||
}
|
||||
if got.OpType != want.OpType {
|
||||
t.Errorf("entry[%d].OpType = %d, want %d", i, got.OpType, want.OpType)
|
||||
}
|
||||
if got.ValueKind != want.ValueKind {
|
||||
t.Errorf("entry[%d].ValueKind = %d, want %d", i, got.ValueKind, want.ValueKind)
|
||||
}
|
||||
if !bytes.Equal(got.Key, want.Key) {
|
||||
t.Errorf("entry[%d].Key = %q, want %q", i, got.Key, want.Key)
|
||||
}
|
||||
if !bytes.Equal(got.Value, want.Value) {
|
||||
t.Errorf("entry[%d].Value = %q, want %q", i, got.Value, want.Value)
|
||||
}
|
||||
offset += consumed
|
||||
}
|
||||
|
||||
if offset != len(batch.Entries) {
|
||||
t.Errorf("parsed %d bytes, Entries region is %d bytes", offset, len(batch.Entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentCollector(t *testing.T) {
|
||||
// Create a batch large enough to span multiple records (~100 KB).
|
||||
var entries []*WalEntry
|
||||
for i := 0; i < 5000; i++ {
|
||||
key := fmt.Sprintf("key_%06d", i)
|
||||
val := fmt.Sprintf("val_%06d_%050s", i, "x") // ~57 bytes value
|
||||
entries = append(entries, makePutEntry(key, val))
|
||||
}
|
||||
|
||||
encoded, err := EncodeWalBatch(1, entries)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeWalBatch: %v", err)
|
||||
}
|
||||
|
||||
// Split into records.
|
||||
records := SplitIntoRecords(encoded)
|
||||
if len(records) < 2 {
|
||||
t.Fatalf("expected multiple records for ~100KB batch, got %d", len(records))
|
||||
}
|
||||
|
||||
// Decode each physical record to get (type, payload).
|
||||
collector := NewFragmentCollector()
|
||||
|
||||
for i, rec := range records {
|
||||
decoded, _, err := DecodePhysicalRecord(rec)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePhysicalRecord[%d]: %v", i, err)
|
||||
}
|
||||
|
||||
// Verify state transitions.
|
||||
switch {
|
||||
case i == 0:
|
||||
if decoded.Type != RecFirst {
|
||||
t.Errorf("record[0] type = %d, want RecFirst(%d)", decoded.Type, RecFirst)
|
||||
}
|
||||
if collector.State() != FragmentIdle {
|
||||
t.Errorf("before append[0] state = %d, want FragmentIdle", collector.State())
|
||||
}
|
||||
case i == len(records)-1:
|
||||
if decoded.Type != RecLast {
|
||||
t.Errorf("record[%d] type = %d, want RecLast(%d)", i, decoded.Type, RecLast)
|
||||
}
|
||||
if collector.State() != FragmentCollecting {
|
||||
t.Errorf("before append[%d] state = %d, want FragmentCollecting", i, collector.State())
|
||||
}
|
||||
default:
|
||||
if decoded.Type != RecMiddle {
|
||||
t.Errorf("record[%d] type = %d, want RecMiddle(%d)", i, decoded.Type, RecMiddle)
|
||||
}
|
||||
if collector.State() != FragmentCollecting {
|
||||
t.Errorf("before append[%d] state = %d, want FragmentCollecting", i, collector.State())
|
||||
}
|
||||
}
|
||||
|
||||
if err := collector.Append(decoded.Type, decoded.Payload); err != nil {
|
||||
t.Fatalf("Append[%d]: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// After all fragments, batch should be complete.
|
||||
if !collector.IsComplete() {
|
||||
t.Fatal("expected IsComplete() after all fragments")
|
||||
}
|
||||
|
||||
// Collected data should match original encoded batch.
|
||||
if !bytes.Equal(collector.BatchData(), encoded) {
|
||||
t.Errorf("collected data length = %d, want %d", len(collector.BatchData()), len(encoded))
|
||||
}
|
||||
|
||||
// Verify the batch can be decoded.
|
||||
batch, err := DecodeWalBatch(collector.BatchData())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeWalBatch from collected: %v", err)
|
||||
}
|
||||
if batch.EntryCount != uint32(len(entries)) {
|
||||
t.Errorf("EntryCount = %d, want %d", batch.EntryCount, len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentCollectorSingleRecord(t *testing.T) {
|
||||
// Small batch fits in one record.
|
||||
entries := []*WalEntry{makePutEntry("k", "v")}
|
||||
encoded, err := EncodeWalBatch(1, entries)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeWalBatch: %v", err)
|
||||
}
|
||||
|
||||
records := SplitIntoRecords(encoded)
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(records))
|
||||
}
|
||||
|
||||
decoded, _, err := DecodePhysicalRecord(records[0])
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePhysicalRecord: %v", err)
|
||||
}
|
||||
if decoded.Type != RecFull {
|
||||
t.Errorf("type = %d, want RecFull(%d)", decoded.Type, RecFull)
|
||||
}
|
||||
|
||||
collector := NewFragmentCollector()
|
||||
if err := collector.Append(decoded.Type, decoded.Payload); err != nil {
|
||||
t.Fatalf("Append: %v", err)
|
||||
}
|
||||
if !collector.IsComplete() {
|
||||
t.Fatal("expected IsComplete()")
|
||||
}
|
||||
if !bytes.Equal(collector.BatchData(), encoded) {
|
||||
t.Error("collected data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentIllegalTransitions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepFunc func(fc *FragmentCollector) // set up initial state
|
||||
recType uint8
|
||||
}{
|
||||
{
|
||||
name: "Idle+RecMiddle",
|
||||
prepFunc: func(fc *FragmentCollector) {},
|
||||
recType: RecMiddle,
|
||||
},
|
||||
{
|
||||
name: "Idle+RecLast",
|
||||
prepFunc: func(fc *FragmentCollector) {},
|
||||
recType: RecLast,
|
||||
},
|
||||
{
|
||||
name: "Collecting+RecFull",
|
||||
prepFunc: func(fc *FragmentCollector) {
|
||||
_ = fc.Append(RecFirst, []byte("data"))
|
||||
},
|
||||
recType: RecFull,
|
||||
},
|
||||
{
|
||||
name: "Collecting+RecFirst",
|
||||
prepFunc: func(fc *FragmentCollector) {
|
||||
_ = fc.Append(RecFirst, []byte("data"))
|
||||
},
|
||||
recType: RecFirst,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fc := NewFragmentCollector()
|
||||
tt.prepFunc(fc)
|
||||
err := fc.Append(tt.recType, []byte("payload"))
|
||||
if err == nil {
|
||||
t.Errorf("expected error for %s, got nil", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentBufferSizeLimit(t *testing.T) {
|
||||
// Exceed the maximum fragment buffer size.
|
||||
fc := NewFragmentCollector()
|
||||
|
||||
// Start collecting.
|
||||
if err := fc.Append(RecFirst, make([]byte, maxFragmentBufferSize-1)); err != nil {
|
||||
t.Fatalf("Append First: %v", err)
|
||||
}
|
||||
if fc.State() != FragmentCollecting {
|
||||
t.Fatal("expected FragmentCollecting state")
|
||||
}
|
||||
|
||||
// This should exceed the limit.
|
||||
err := fc.Append(RecMiddle, make([]byte, 10))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for exceeding buffer size limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragmentReset(t *testing.T) {
|
||||
fc := NewFragmentCollector()
|
||||
if err := fc.Append(RecFirst, []byte("data")); err != nil {
|
||||
t.Fatalf("Append: %v", err)
|
||||
}
|
||||
if fc.State() != FragmentCollecting {
|
||||
t.Fatal("expected FragmentCollecting")
|
||||
}
|
||||
|
||||
fc.Reset()
|
||||
if fc.State() != FragmentIdle {
|
||||
t.Errorf("state after Reset = %d, want FragmentIdle", fc.State())
|
||||
}
|
||||
if fc.IsComplete() {
|
||||
t.Error("IsComplete() should be false after Reset")
|
||||
}
|
||||
if fc.buf.Len() != 0 {
|
||||
t.Errorf("buffer len after Reset = %d, want 0", fc.buf.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchValidationErrors(t *testing.T) {
|
||||
t.Run("EntryCountZero", func(t *testing.T) {
|
||||
// Manually craft a batch with EntryCount=0.
|
||||
buf := make([]byte, WalBatchHeaderSize)
|
||||
binary.LittleEndian.PutUint16(buf[0:2], 0)
|
||||
binary.LittleEndian.PutUint64(buf[2:10], 1)
|
||||
binary.LittleEndian.PutUint32(buf[10:14], 0) // entryCount = 0
|
||||
binary.LittleEndian.PutUint32(buf[14:18], 0)
|
||||
_, err := DecodeWalBatch(buf)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for entryCount=0")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("EntryCountExceedsMax", func(t *testing.T) {
|
||||
buf := make([]byte, WalBatchHeaderSize)
|
||||
binary.LittleEndian.PutUint16(buf[0:2], 0)
|
||||
binary.LittleEndian.PutUint64(buf[2:10], 1)
|
||||
binary.LittleEndian.PutUint32(buf[10:14], MaxWalBatchEntryCount+1)
|
||||
binary.LittleEndian.PutUint32(buf[14:18], 1)
|
||||
_, err := DecodeWalBatch(buf)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for entryCount > max")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("EntriesSizeMismatch", func(t *testing.T) {
|
||||
buf := make([]byte, WalBatchHeaderSize)
|
||||
binary.LittleEndian.PutUint16(buf[0:2], 0)
|
||||
binary.LittleEndian.PutUint64(buf[2:10], 1)
|
||||
binary.LittleEndian.PutUint32(buf[10:14], 1)
|
||||
binary.LittleEndian.PutUint32(buf[14:18], 100) // entriesSize=100 but no data after header
|
||||
_, err := DecodeWalBatch(buf)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for entriesSize mismatch")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DataTooShort", func(t *testing.T) {
|
||||
_, err := DecodeWalBatch([]byte{1, 2, 3})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for data too short")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("EmptyEntriesSlice", func(t *testing.T) {
|
||||
_, err := EncodeWalBatch(1, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty entries slice")
|
||||
}
|
||||
})
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/dailz/go-kv/config"
|
||||
)
|
||||
|
||||
// ValidateBatchLimits checks that a batch of WAL entries satisfies all resource
|
||||
// limits from the supplied configuration before sequence allocation or WAL append.
|
||||
// Returns a descriptive error on violation, nil on success.
|
||||
func ValidateBatchLimits(entries []*WalEntry, cfg *config.WalConfig) error {
|
||||
entryCount := uint64(len(entries))
|
||||
if entryCount == 0 {
|
||||
return fmt.Errorf("wal: batch entry count must be > 0")
|
||||
}
|
||||
if entryCount > uint64(cfg.MaxBatchEntries) {
|
||||
return fmt.Errorf("wal: entry count %d exceeds limit %d", entryCount, cfg.MaxBatchEntries)
|
||||
}
|
||||
|
||||
// Per-entry validation and total encoded size calculation.
|
||||
// Each entry encoded size = 1(opType) + 1(valueKind) + varint(keyLen) + varint(valLen) + keyLen + valLen
|
||||
// We use MaxWalVarintBytes (5) as worst-case varint size.
|
||||
var totalEncodedSize uint64
|
||||
for i, e := range entries {
|
||||
keyLen := uint64(len(e.Key))
|
||||
if keyLen == 0 || keyLen > uint64(cfg.MaxKeyBytes) {
|
||||
return fmt.Errorf("wal: entry %d: key length %d out of range [1, %d]", i, keyLen, cfg.MaxKeyBytes)
|
||||
}
|
||||
valLen := uint64(len(e.Value))
|
||||
|
||||
if e.OpType == OpPut && e.ValueKind == VKInline {
|
||||
if valLen > uint64(cfg.MaxInlineValue) {
|
||||
return fmt.Errorf("wal: entry %d: inline value length %d exceeds limit %d", i, valLen, cfg.MaxInlineValue)
|
||||
}
|
||||
}
|
||||
|
||||
// entrySize = 2 + varint(keyLen) + varint(valLen) + keyLen + valLen
|
||||
// Use worst-case varint size for safety.
|
||||
entrySize, err := safeAddChecked(2+MaxWalVarintBytes+MaxWalVarintBytes, keyLen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: entry %d: size overflow: %w", i, err)
|
||||
}
|
||||
entrySize, err = safeAddChecked(entrySize, valLen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: entry %d: size overflow: %w", i, err)
|
||||
}
|
||||
totalEncodedSize, err = safeAddChecked(totalEncodedSize, entrySize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: total encoded size overflow: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// totalWithBatchHeader = WalBatchHeaderSize + totalEncodedSize
|
||||
totalWithBatchHeader, err := safeAddChecked(WalBatchHeaderSize, totalEncodedSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: batch size overflow: %w", err)
|
||||
}
|
||||
|
||||
if totalWithBatchHeader > uint64(cfg.MaxBatchSize) {
|
||||
return fmt.Errorf("wal: total batch size %d exceeds limit %d", totalWithBatchHeader, cfg.MaxBatchSize)
|
||||
}
|
||||
|
||||
// Check that the batch fits in a WAL segment with physical record overhead.
|
||||
blockSize := uint64(cfg.BlockSize)
|
||||
prHeaderSize := uint64(PhysicalRecordHeaderSize)
|
||||
|
||||
if blockSize <= prHeaderSize {
|
||||
return fmt.Errorf("wal: block size %d must be > physical record header size %d", blockSize, prHeaderSize)
|
||||
}
|
||||
maxPayload := blockSize - prHeaderSize
|
||||
|
||||
numRecords := divCeilChecked(totalWithBatchHeader, maxPayload)
|
||||
overhead, err := safeMulChecked(numRecords, prHeaderSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: physical record overhead overflow: %w", err)
|
||||
}
|
||||
|
||||
totalWithOverhead, err := safeAddChecked(totalWithBatchHeader, overhead)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wal: total with overhead overflow: %w", err)
|
||||
}
|
||||
|
||||
maxSegmentPayload := cfg.MaxSegmentSize - WalFileHeaderSize
|
||||
if totalWithOverhead > maxSegmentPayload {
|
||||
return fmt.Errorf("wal: batch with overhead %d exceeds segment payload %d", totalWithOverhead, maxSegmentPayload)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// safeAddChecked returns a + b or an error if the result overflows uint64.
|
||||
func safeAddChecked(a, b uint64) (uint64, error) {
|
||||
if a > math.MaxUint64-b {
|
||||
return 0, fmt.Errorf("uint64 overflow: %d + %d", a, b)
|
||||
}
|
||||
return a + b, nil
|
||||
}
|
||||
|
||||
// safeMulChecked returns a * b or an error if the result overflows uint64.
|
||||
func safeMulChecked(a, b uint64) (uint64, error) {
|
||||
if a != 0 && b > math.MaxUint64/a {
|
||||
return 0, fmt.Errorf("uint64 overflow: %d * %d", a, b)
|
||||
}
|
||||
return a * b, nil
|
||||
}
|
||||
|
||||
// divCeilChecked returns ceil(a / b) for b > 0.
|
||||
func divCeilChecked(a, b uint64) uint64 {
|
||||
return (a + b - 1) / b
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dailz/go-kv/config"
|
||||
)
|
||||
|
||||
func makeEntry(opType uint8, valueKind uint8, keySize, valSize int) *WalEntry {
|
||||
key := make([]byte, keySize)
|
||||
for i := range key {
|
||||
key[i] = byte('a' + i%26)
|
||||
}
|
||||
val := make([]byte, valSize)
|
||||
for i := range val {
|
||||
val[i] = byte('x')
|
||||
}
|
||||
return &WalEntry{OpType: opType, ValueKind: valueKind, Key: key, Value: val}
|
||||
}
|
||||
|
||||
func TestValidateBatchLimitsPass(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
entries := make([]*WalEntry, 100)
|
||||
for i := range entries {
|
||||
entries[i] = makeEntry(OpPut, VKInline, 10, 10)
|
||||
}
|
||||
if err := ValidateBatchLimits(entries, &cfg); err != nil {
|
||||
t.Fatalf("expected nil error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateEntryCountExceeded(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
entries := make([]*WalEntry, 10001)
|
||||
for i := range entries {
|
||||
entries[i] = makeEntry(OpPut, VKInline, 10, 10)
|
||||
}
|
||||
err := ValidateBatchLimits(entries, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for entry count exceeded")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "entry count") {
|
||||
t.Fatalf("expected error containing 'entry count', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateKeyTooBig(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
// 5KB key exceeds 4KB limit
|
||||
entries := []*WalEntry{makeEntry(OpPut, VKInline, 5*1024, 10)}
|
||||
err := ValidateBatchLimits(entries, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for key too big")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "key") {
|
||||
t.Fatalf("expected error containing 'key', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateValueTooBig(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
// 5KB inline value exceeds 4KB limit
|
||||
entries := []*WalEntry{makeEntry(OpPut, VKInline, 10, 5*1024)}
|
||||
err := ValidateBatchLimits(entries, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for value too big")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "value") {
|
||||
t.Fatalf("expected error containing 'value', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateEmptyBatch(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
err := ValidateBatchLimits(nil, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty batch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "entry count") && !strings.Contains(err.Error(), "> 0") {
|
||||
t.Fatalf("expected error about empty batch, got: %v", err)
|
||||
}
|
||||
|
||||
err = ValidateBatchLimits([]*WalEntry{}, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty batch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTotalSizeExceeded(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
// Create entries that collectively exceed MaxBatchSize (4MB).
|
||||
// Each entry with 1000-byte key + 1000-byte value ≈ 2012 bytes encoded.
|
||||
// 2100 entries × ~2012 ≈ ~4.2MB > 4MB
|
||||
entries := make([]*WalEntry, 2100)
|
||||
for i := range entries {
|
||||
entries[i] = makeEntry(OpPut, VKInline, 1000, 1000)
|
||||
}
|
||||
err := ValidateBatchLimits(entries, &cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for total size exceeded")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "batch size") {
|
||||
t.Fatalf("expected error containing 'batch size', got: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user