feat: initialize project structure and error types
- go.mod with github.com/dailz/go-kv, Go 1.26.3, testify - config/config.go with WalConfig, Validate() with checked arithmetic - errors.go with sentinel errors (ErrCommitUnknown, ErrWriteStopped, etc.) - wal/constants.go with all WAL format constants and enums - wal/header.go with WAL File Header encode/decode (CRC32 IEEE) - wal/record.go with Physical Record codec, block boundary, SplitIntoRecords - wal/entry.go with WAL Entry codec (varint keys/values, OpType, ValueKind) - wal/sequence.go with SequenceManager (atomic, CAS, overflow-safe) - manifest/manifest.go with MANIFEST stub (Load/Save atomic) - manifest/current.go with CURRENT file (WriteCurrent/ReadCurrent) - Comprehensive tests for all modules - .golangci.yml configuration
This commit is contained in:
dailz
@@ -0,0 +1,205 @@
|
||||
// Package config defines configuration types and validation for the go-kv storage engine.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
)
|
||||
|
||||
// WAL format constants derived from the binary layout specification.
|
||||
const (
|
||||
walFileHeaderSize uint64 = 32
|
||||
walBlockSize uint32 = 32 * 1024 // 32KB
|
||||
physicalRecordHeaderSize uint64 = 7
|
||||
walBatchHeaderSize uint64 = 18 // flags(2) + baseSequence(8) + entryCount(4) + entriesSize(4)
|
||||
)
|
||||
|
||||
// WalConfig holds configuration for the Write-Ahead Log subsystem.
|
||||
// Zero-value WalConfig is valid and uses defaults; call Validate() to apply
|
||||
// defaults and verify invariants.
|
||||
type WalConfig struct {
|
||||
// MaxSegmentSize is the maximum size of a single WAL segment file in bytes.
|
||||
// Default: 64MB. Must be large enough to hold the largest possible WAL Batch.
|
||||
MaxSegmentSize uint64
|
||||
|
||||
// BlockSize is the WAL block size in bytes.
|
||||
// Default: 32KB.
|
||||
BlockSize uint32
|
||||
|
||||
// SyncMode controls when WAL writes are flushed to disk.
|
||||
// Phase 1 only supports "always".
|
||||
SyncMode string
|
||||
|
||||
// MaxBatchEntries is the maximum number of entries in a single WAL Batch.
|
||||
// Default: 10000.
|
||||
MaxBatchEntries uint32
|
||||
|
||||
// MaxBatchSize is the maximum total size of WAL Batch entries in bytes.
|
||||
// Default: 4MB.
|
||||
MaxBatchSize uint32
|
||||
|
||||
// MaxKeyBytes is the maximum size of a single key in bytes.
|
||||
// Default: 4KB.
|
||||
MaxKeyBytes uint32
|
||||
|
||||
// MaxInlineValue is the maximum size of an inline value in bytes.
|
||||
// Values larger than this must use ValueLogPointer.
|
||||
// Default: 4KB.
|
||||
MaxInlineValue uint32
|
||||
|
||||
// MemTableSize is the target MemTable size in bytes before triggering flush.
|
||||
// Default: 64MB.
|
||||
MemTableSize uint32
|
||||
|
||||
// MaxImmutableCount is the maximum number of immutable MemTables allowed
|
||||
// before writes are stalled. Default: 3.
|
||||
MaxImmutableCount int
|
||||
}
|
||||
|
||||
// Defaults returns a WalConfig populated with production defaults.
|
||||
func Defaults() WalConfig {
|
||||
return WalConfig{
|
||||
MaxSegmentSize: 64 * 1024 * 1024, // 64MB
|
||||
BlockSize: 32 * 1024, // 32KB
|
||||
SyncMode: "always",
|
||||
MaxBatchEntries: 10000,
|
||||
MaxBatchSize: 4 * 1024 * 1024, // 4MB
|
||||
MaxKeyBytes: 4 * 1024, // 4KB
|
||||
MaxInlineValue: 4 * 1024, // 4KB
|
||||
MemTableSize: 64 * 1024 * 1024, // 64MB
|
||||
MaxImmutableCount: 3,
|
||||
}
|
||||
}
|
||||
|
||||
// applyDefaults fills zero-valued fields with production defaults.
|
||||
func (c *WalConfig) applyDefaults() {
|
||||
d := Defaults()
|
||||
if c.MaxSegmentSize == 0 {
|
||||
c.MaxSegmentSize = d.MaxSegmentSize
|
||||
}
|
||||
if c.BlockSize == 0 {
|
||||
c.BlockSize = d.BlockSize
|
||||
}
|
||||
if c.SyncMode == "" {
|
||||
c.SyncMode = d.SyncMode
|
||||
}
|
||||
if c.MaxBatchEntries == 0 {
|
||||
c.MaxBatchEntries = d.MaxBatchEntries
|
||||
}
|
||||
if c.MaxBatchSize == 0 {
|
||||
c.MaxBatchSize = d.MaxBatchSize
|
||||
}
|
||||
if c.MaxKeyBytes == 0 {
|
||||
c.MaxKeyBytes = d.MaxKeyBytes
|
||||
}
|
||||
if c.MaxInlineValue == 0 {
|
||||
c.MaxInlineValue = d.MaxInlineValue
|
||||
}
|
||||
if c.MemTableSize == 0 {
|
||||
c.MemTableSize = d.MemTableSize
|
||||
}
|
||||
if c.MaxImmutableCount == 0 {
|
||||
c.MaxImmutableCount = d.MaxImmutableCount
|
||||
}
|
||||
}
|
||||
|
||||
// Validate applies defaults and verifies that all configuration invariants hold.
|
||||
// The key invariant ensures that the largest possible WAL Batch can fit into
|
||||
// an empty WAL segment:
|
||||
//
|
||||
// maxWalSegmentPayload >= maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding
|
||||
//
|
||||
// All arithmetic is checked for overflow.
|
||||
func (c *WalConfig) Validate() error {
|
||||
c.applyDefaults()
|
||||
|
||||
if c.SyncMode != "always" {
|
||||
return fmt.Errorf("config: SyncMode %q not supported (Phase 1: only \"always\")", c.SyncMode)
|
||||
}
|
||||
|
||||
if c.MaxImmutableCount < 1 {
|
||||
return fmt.Errorf("config: MaxImmutableCount must be >= 1, got %d", c.MaxImmutableCount)
|
||||
}
|
||||
|
||||
// --- Checked arithmetic invariant validation ---
|
||||
// Mirrors the derivation in docs/design.md § WAL Segment Rotation.
|
||||
|
||||
blockSize := uint64(c.BlockSize)
|
||||
prHeaderSize := physicalRecordHeaderSize
|
||||
batchHeaderSize := walBatchHeaderSize
|
||||
maxBatchEntriesSize := uint64(c.MaxBatchSize)
|
||||
|
||||
// maxEncodedWalBatchSize = batchHeaderSize + maxBatchEntriesSize
|
||||
maxEncodedWalBatchSize, err := safeAdd(batchHeaderSize, maxBatchEntriesSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config: WAL batch size overflow: %w", err)
|
||||
}
|
||||
|
||||
// maxPhysicalRecordPayload = blockSize - prHeaderSize
|
||||
if blockSize <= prHeaderSize {
|
||||
return fmt.Errorf("config: BlockSize %d must be > physical record header size %d", blockSize, prHeaderSize)
|
||||
}
|
||||
maxPhysicalRecordPayload := blockSize - prHeaderSize
|
||||
|
||||
// maxPhysicalRecordCount = ceil(maxEncodedWalBatchSize / maxPhysicalRecordPayload)
|
||||
maxPhysicalRecordCount := divCeil(maxEncodedWalBatchSize, maxPhysicalRecordPayload)
|
||||
|
||||
// worstCasePhysicalRecordOverhead = maxPhysicalRecordCount * prHeaderSize
|
||||
worstCasePhysicalRecordOverhead, err := safeMul(maxPhysicalRecordCount, prHeaderSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config: physical record overhead overflow: %w", err)
|
||||
}
|
||||
|
||||
// worstCaseBlockPadding = blockSize - 1 (at most one partial block of padding)
|
||||
// From design doc: worstCaseBlockPadding = 7 bytes with default block size.
|
||||
// Generalized: blockSize - maxPhysicalRecordPayload = prHeaderSize
|
||||
worstCaseBlockPadding := prHeaderSize
|
||||
|
||||
// minWalSegmentPayload = maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding
|
||||
partial, err := safeAdd(maxEncodedWalBatchSize, worstCasePhysicalRecordOverhead)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config: segment payload calculation overflow: %w", err)
|
||||
}
|
||||
minWalSegmentPayload, err := safeAdd(partial, worstCaseBlockPadding)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config: segment payload calculation overflow: %w", err)
|
||||
}
|
||||
|
||||
// maxWalSegmentPayload = MaxSegmentSize - walFileHeaderSize
|
||||
if c.MaxSegmentSize <= walFileHeaderSize {
|
||||
return fmt.Errorf("config: MaxSegmentSize %d must be > WAL file header size %d",
|
||||
c.MaxSegmentSize, walFileHeaderSize)
|
||||
}
|
||||
maxWalSegmentPayload := c.MaxSegmentSize - walFileHeaderSize
|
||||
|
||||
if maxWalSegmentPayload < minWalSegmentPayload {
|
||||
return fmt.Errorf("config: MaxSegmentSize %d too small: "+
|
||||
"segment payload (%d) < minimum required (%d); "+
|
||||
"need MaxSegmentSize >= %d",
|
||||
c.MaxSegmentSize, maxWalSegmentPayload, minWalSegmentPayload,
|
||||
minWalSegmentPayload+walFileHeaderSize)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// safeAdd returns a + b or an error if the result overflows uint64.
|
||||
func safeAdd(a, b uint64) (uint64, error) {
|
||||
if a > math.MaxUint64-b {
|
||||
return 0, fmt.Errorf("uint64 overflow: %d + %d", a, b)
|
||||
}
|
||||
return a + b, nil
|
||||
}
|
||||
|
||||
// safeMul returns a * b or an error if the result overflows uint64.
|
||||
func safeMul(a, b uint64) (uint64, error) {
|
||||
if a != 0 && b > math.MaxUint64/a {
|
||||
return 0, fmt.Errorf("uint64 overflow: %d * %d", a, b)
|
||||
}
|
||||
return a * b, nil
|
||||
}
|
||||
|
||||
// divCeil returns ceil(a / b) for b > 0.
|
||||
func divCeil(a, b uint64) uint64 {
|
||||
return (a + b - 1) / b
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestValidateDefaults(t *testing.T) {
|
||||
cfg := WalConfig{}
|
||||
err := cfg.Validate()
|
||||
require.NoError(t, err, "default WalConfig should pass validation")
|
||||
|
||||
d := Defaults()
|
||||
assert.Equal(t, d.MaxSegmentSize, cfg.MaxSegmentSize, "MaxSegmentSize should be defaulted")
|
||||
assert.Equal(t, d.BlockSize, cfg.BlockSize, "BlockSize should be defaulted")
|
||||
assert.Equal(t, d.SyncMode, cfg.SyncMode, "SyncMode should be defaulted")
|
||||
assert.Equal(t, d.MaxBatchEntries, cfg.MaxBatchEntries, "MaxBatchEntries should be defaulted")
|
||||
assert.Equal(t, d.MaxBatchSize, cfg.MaxBatchSize, "MaxBatchSize should be defaulted")
|
||||
assert.Equal(t, d.MaxKeyBytes, cfg.MaxKeyBytes, "MaxKeyBytes should be defaulted")
|
||||
assert.Equal(t, d.MaxInlineValue, cfg.MaxInlineValue, "MaxInlineValue should be defaulted")
|
||||
assert.Equal(t, d.MemTableSize, cfg.MemTableSize, "MemTableSize should be defaulted")
|
||||
assert.Equal(t, d.MaxImmutableCount, cfg.MaxImmutableCount, "MaxImmutableCount should be defaulted")
|
||||
}
|
||||
|
||||
func TestValidateSegmentTooSmall(t *testing.T) {
|
||||
cfg := WalConfig{MaxSegmentSize: 1024}
|
||||
err := cfg.Validate()
|
||||
require.Error(t, err, "MaxSegmentSize=1024 should fail validation")
|
||||
assert.Contains(t, err.Error(), "too small")
|
||||
}
|
||||
Reference in New Issue
Block a user