feat(wal): implement batch codec, validation, and Arena allocator
- wal/batch.go: WalBatch encode/decode with FragmentCollector state machine - wal/validate.go: ValidateBatchLimits with checked arithmetic - memtable/arena.go: Arena allocator with 8-byte alignment and mutex - Comprehensive tests for all modules, all pass with -race
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
package memtable
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// ErrArenaFull is returned when the arena cannot satisfy an allocation request.
|
||||
var ErrArenaFull = errors.New("arena: insufficient capacity")
|
||||
|
||||
// Arena is a bump allocator backed by a fixed-size byte slice.
|
||||
// It provides thread-safe allocation with 8-byte alignment.
|
||||
//
|
||||
// The arena is used by the memtable to store keys, values, and skip-list
|
||||
// node structures. Once allocated, bytes are never freed — the entire
|
||||
// arena is discarded when the memtable is flushed.
|
||||
type Arena struct {
|
||||
buf []byte
|
||||
offset uint32 // next allocation offset (atomic for reads)
|
||||
capacity uint32 // total capacity (immutable)
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewArena creates a new Arena with the given capacity in bytes.
|
||||
// The entire buffer is allocated upfront.
|
||||
func NewArena(capacity uint32) *Arena {
|
||||
return &Arena{
|
||||
buf: make([]byte, capacity),
|
||||
capacity: capacity,
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate reserves alignedSize bytes in the arena and returns the offset
|
||||
// at which the caller can write. The size is aligned up to 8 bytes.
|
||||
func (a *Arena) Allocate(size uint32) (uint32, error) {
|
||||
alignedSize := (size + 7) &^ uint32(7)
|
||||
|
||||
a.mu.Lock()
|
||||
rem := a.capacity - a.offset
|
||||
if rem < alignedSize {
|
||||
a.mu.Unlock()
|
||||
return 0, ErrArenaFull
|
||||
}
|
||||
off := a.offset
|
||||
a.offset += alignedSize
|
||||
a.mu.Unlock()
|
||||
|
||||
return off, nil
|
||||
}
|
||||
|
||||
// GetBytes returns a slice of the arena buffer at [offset, offset+size).
|
||||
// It panics if the range is out of bounds.
|
||||
func (a *Arena) GetBytes(offset, size uint32) []byte {
|
||||
end := offset + size
|
||||
if end > a.capacity {
|
||||
panic("arena: GetBytes out of bounds")
|
||||
}
|
||||
return a.buf[offset:end]
|
||||
}
|
||||
|
||||
// Remaining returns the number of bytes still available for allocation.
|
||||
func (a *Arena) Remaining() uint32 {
|
||||
return a.capacity - atomic.LoadUint32(&a.offset)
|
||||
}
|
||||
|
||||
// Capacity returns the total capacity of the arena.
|
||||
func (a *Arena) Capacity() uint32 {
|
||||
return a.capacity
|
||||
}
|
||||
|
||||
// Reserve checks whether the arena has at least totalSize bytes remaining
|
||||
// without actually allocating. It is used by the WAL writer to verify
|
||||
// capacity before appending entries.
|
||||
func (a *Arena) Reserve(totalSize uint32) error {
|
||||
a.mu.Lock()
|
||||
rem := a.capacity - a.offset
|
||||
a.mu.Unlock()
|
||||
|
||||
if rem < totalSize {
|
||||
return ErrArenaFull
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package memtable
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestArenaAllocate(t *testing.T) {
|
||||
a := NewArena(1024)
|
||||
|
||||
off, err := a.Allocate(9)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint32(0), off)
|
||||
|
||||
copy(a.GetBytes(off, 9), "test data")
|
||||
require.Equal(t, []byte("test data"), a.GetBytes(off, 9))
|
||||
|
||||
off2, err := a.Allocate(200)
|
||||
require.NoError(t, err)
|
||||
// 9 bytes aligned to 16, so second allocation starts at 16
|
||||
require.Equal(t, uint32(16), off2)
|
||||
}
|
||||
|
||||
func TestArenaFull(t *testing.T) {
|
||||
a := NewArena(64)
|
||||
|
||||
off, err := a.Allocate(60)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint32(0), off) // 60 aligned to 64
|
||||
|
||||
_, err = a.Allocate(10)
|
||||
require.ErrorIs(t, err, ErrArenaFull)
|
||||
}
|
||||
|
||||
func TestArenaAlignment(t *testing.T) {
|
||||
a := NewArena(1024)
|
||||
|
||||
sizes := []uint32{1, 3, 5, 7, 8, 9, 13, 16, 100}
|
||||
for _, s := range sizes {
|
||||
off, err := a.Allocate(s)
|
||||
require.NoError(t, err, "size=%d", s)
|
||||
require.Equal(t, uint32(0), off%8, "offset %d not 8-byte aligned for size %d", off, s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArenaConcurrent(t *testing.T) {
|
||||
a := NewArena(65536)
|
||||
|
||||
const goroutines = 64
|
||||
const perGoroutine = 32
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
|
||||
offsets := make([][]uint32, goroutines)
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
locals := make([]uint32, 0, perGoroutine)
|
||||
for j := 0; j < perGoroutine; j++ {
|
||||
off, err := a.Allocate(8)
|
||||
if err == nil {
|
||||
// Write unique data to detect corruption
|
||||
copy(a.GetBytes(off, 8), []byte{byte(i), byte(j), 0, 0, 0, 0, 0, 0})
|
||||
locals = append(locals, off)
|
||||
}
|
||||
}
|
||||
offsets[i] = locals
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Verify data integrity — each byte pair must match its goroutine/j index
|
||||
for i, locals := range offsets {
|
||||
for j, off := range locals {
|
||||
data := a.GetBytes(off, 8)
|
||||
require.Equal(t, byte(i), data[0], "goroutine %d offset %d", i, off)
|
||||
require.Equal(t, byte(j), data[1], "iteration %d offset %d", j, off)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArenaReserve(t *testing.T) {
|
||||
a := NewArena(128)
|
||||
|
||||
require.NoError(t, a.Reserve(64))
|
||||
require.NoError(t, a.Reserve(128))
|
||||
require.ErrorIs(t, a.Reserve(129), ErrArenaFull)
|
||||
|
||||
// Allocate some space
|
||||
_, err := a.Allocate(32)
|
||||
require.NoError(t, err) // 32 aligned = 32
|
||||
|
||||
// 128 - 32 = 96 remaining
|
||||
require.NoError(t, a.Reserve(96))
|
||||
require.ErrorIs(t, a.Reserve(97), ErrArenaFull)
|
||||
}
|
||||
|
||||
func TestArenaRemainingAndCapacity(t *testing.T) {
|
||||
cap := uint32(256)
|
||||
a := NewArena(cap)
|
||||
|
||||
require.Equal(t, cap, a.Capacity())
|
||||
require.Equal(t, cap, a.Remaining())
|
||||
|
||||
_, err := a.Allocate(10)
|
||||
require.NoError(t, err) // 10 aligned to 16
|
||||
|
||||
require.Equal(t, cap-16, a.Remaining())
|
||||
}
|
||||
Reference in New Issue
Block a user