feat(wal,memtable): implement segment writer, block writer, and MemTable

- wal/block_writer.go: 32KB block buffer with padding and flush - wal/segment_writer.go: WAL segment file with durable-ready protocol - memtable/memtable.go: Arena+SkipList wrapper with publish/abort semantics - Comprehensive tests for all modules, all pass with -race
2026-06-12 13:43:24 +08:00
parent 3c0e3b14ff
commit 349063968b
8 changed files with 1407 additions and 4 deletions
@@ -0,0 +1,159 @@
+package memtable
+
+import (
+	"errors"
+	"sync"
+	"sync/atomic"
+)
+
+// ErrMemTableFull is returned when the memtable cannot reserve enough space.
+var ErrMemTableFull = errors.New("memtable: insufficient capacity")
+
+// ReserveEntry describes a single key-value pair to be reserved.
+type ReserveEntry struct {
+	Key      []byte
+	Value    []byte
+	IsDelete bool
+}
+
+// GetResult is returned by MemTable.Get.
+type GetResult struct {
+	Found    bool
+	Value    []byte
+	Sequence uint64
+}
+
+// MemTable wraps an Arena and SkipList with publish/abort semantics.
+//
+// Writers first reserve capacity, then insert pending entries, then either
+// publish (making them visible) or abort (leaving them invisible forever).
+// Readers only see published entries that have not been aborted.
+type MemTable struct {
+	arena     *Arena
+	skiplist  *SkipList
+	published atomic.Uint64 // highest published sequence
+	aborted   sync.Map      // map[uint64]struct{} — set of aborted sequences
+}
+
+// NewMemTable creates a MemTable backed by an arena of the given capacity.
+func NewMemTable(capacity uint32) *MemTable {
+	arena := NewArena(capacity)
+	return &MemTable{
+		arena:    arena,
+		skiplist: NewSkipList(arena),
+	}
+}
+
+// Reserve estimates whether the arena has enough space for all entries.
+// It returns the total estimated size for caller tracking.
+//
+// The estimate is conservative: key + value bytes plus a fixed metadata
+// overhead per entry. Since Phase 1 stores nodes on the Go heap (not in
+// the arena), this reservation is primarily a capacity gate for future
+// arena-backed phases.
+func (mt *MemTable) Reserve(entries []ReserveEntry) (uint32, error) {
+	const metadataOverhead uint32 = 32 // per-entry overhead estimate
+
+	var totalSize uint32
+	for _, e := range entries {
+		sz := metadataOverhead + uint32(len(e.Key)) + uint32(len(e.Value))
+		// Align to 8 bytes
+		sz = (sz + 7) &^ uint32(7)
+		totalSize += sz
+	}
+
+	if err := mt.arena.Reserve(totalSize); err != nil {
+		return 0, ErrMemTableFull
+	}
+	return totalSize, nil
+}
+
+// PutPending inserts a key-value pair as pending (invisible to Get/Iterator).
+func (mt *MemTable) PutPending(key []byte, value []byte, sequence uint64) error {
+	return mt.skiplist.Put(key, value, sequence, true)
+}
+
+// DeletePending inserts a tombstone entry (nil value) as pending.
+func (mt *MemTable) DeletePending(key []byte, sequence uint64) error {
+	return mt.skiplist.Put(key, nil, sequence, true)
+}
+
+// Publish makes all pending entries with sequence <= upToSequence visible.
+//
+// It iterates the skip list and re-inserts each pending entry with
+// pending=false, which the lock-free reader path will then observe.
+// Aborted entries are skipped and remain invisible forever.
+func (mt *MemTable) Publish(upToSequence uint64) {
+	// Collect entries to publish under the iterator (which skips pending).
+	// We need a raw walk, so we access the skiplist directly.
+	type entry struct {
+		key      []byte
+		value    []byte
+		sequence uint64
+	}
+
+	// Walk the raw skip list level-0 chain to find pending entries.
+	// We cannot use NewIterator because it skips pending entries.
+	var toPublish []entry
+
+	mt.skiplist.mu.Lock()
+	node := mt.skiplist.head.next[0].Load()
+	for node != nil {
+		if node.pending && node.sequence <= upToSequence {
+			if _, aborted := mt.aborted.Load(node.sequence); !aborted {
+				toPublish = append(toPublish, entry{
+					key:      node.key,
+					value:    node.value,
+					sequence: node.sequence,
+				})
+			}
+		}
+		node = node.next[0].Load()
+	}
+	mt.skiplist.mu.Unlock()
+
+	// Re-put each entry as published. Each call acquires the skiplist mutex.
+	for _, e := range toPublish {
+		_ = mt.skiplist.Put(e.key, e.value, e.sequence, false)
+	}
+
+	// Update high-water mark after all entries are visible.
+	mt.published.Store(upToSequence)
+}
+
+// Abort marks a sequence as aborted. Aborted entries are never visible to readers.
+func (mt *MemTable) Abort(sequence uint64) {
+	mt.aborted.Store(sequence, struct{}{})
+}
+
+// Get retrieves the latest visible value for key.
+//
+// A value is visible if it is published (pending=false in the skip list)
+// and not in the aborted set.
+func (mt *MemTable) Get(key []byte) *GetResult {
+	found, value, sequence := mt.skiplist.Get(key)
+	if !found {
+		return &GetResult{Found: false}
+	}
+	return &GetResult{
+		Found:    true,
+		Value:    value,
+		Sequence: sequence,
+	}
+}
+
+// NewIterator returns a forward iterator over all published (visible) entries.
+// Pending and aborted entries are automatically skipped.
+func (mt *MemTable) NewIterator() *Iterator {
+	return mt.skiplist.NewIterator()
+}
+
+// ApproximateSize returns the number of bytes used in the arena.
+func (mt *MemTable) ApproximateSize() uint64 {
+	return uint64(mt.arena.Capacity() - mt.arena.Remaining())
+}
+
+// UsableCapacity returns the remaining bytes available in the arena.
+func (mt *MemTable) UsableCapacity() uint32 {
+	return mt.arena.Remaining()
+}
@@ -0,0 +1,209 @@
+package memtable
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestMemTablePendingInvisible(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	// Put a pending entry — must not be visible.
+	if err := mt.PutPending([]byte("key1"), []byte("val1"), 1); err != nil {
+		t.Fatalf("PutPending: %v", err)
+	}
+
+	res := mt.Get([]byte("key1"))
+	if res.Found {
+		t.Fatal("pending entry should not be visible")
+	}
+
+	// Publish sequence 1 — entry becomes visible.
+	mt.Publish(1)
+
+	res = mt.Get([]byte("key1"))
+	if !res.Found {
+		t.Fatal("published entry should be visible")
+	}
+	if string(res.Value) != "val1" {
+		t.Fatalf("value mismatch: got %q, want %q", res.Value, "val1")
+	}
+	if res.Sequence != 1 {
+		t.Fatalf("sequence mismatch: got %d, want %d", res.Sequence, 1)
+	}
+}
+
+func TestMemTableAbortedInvisible(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	if err := mt.PutPending([]byte("key1"), []byte("val1"), 1); err != nil {
+		t.Fatalf("PutPending: %v", err)
+	}
+
+	// Abort sequence 1 before publishing.
+	mt.Abort(1)
+
+	// Publish up to sequence 10 — aborted entry should stay invisible.
+	mt.Publish(10)
+
+	res := mt.Get([]byte("key1"))
+	if res.Found {
+		t.Fatal("aborted entry should not be visible")
+	}
+}
+
+func TestMemTableMultiplePublish(t *testing.T) {
+	mt := NewMemTable(8192)
+
+	if err := mt.PutPending([]byte("a"), []byte("va"), 1); err != nil {
+		t.Fatalf("PutPending a: %v", err)
+	}
+	if err := mt.PutPending([]byte("b"), []byte("vb"), 2); err != nil {
+		t.Fatalf("PutPending b: %v", err)
+	}
+
+	// Neither visible before publish.
+	if mt.Get([]byte("a")).Found {
+		t.Fatal("a should not be visible before publish")
+	}
+	if mt.Get([]byte("b")).Found {
+		t.Fatal("b should not be visible before publish")
+	}
+
+	// Publish both.
+	mt.Publish(2)
+
+	if !mt.Get([]byte("a")).Found {
+		t.Fatal("a should be visible after publish")
+	}
+	if !mt.Get([]byte("b")).Found {
+		t.Fatal("b should be visible after publish")
+	}
+
+	if string(mt.Get([]byte("a")).Value) != "va" {
+		t.Fatal("value mismatch for a")
+	}
+	if string(mt.Get([]byte("b")).Value) != "vb" {
+		t.Fatal("value mismatch for b")
+	}
+}
+
+func TestMemTableReserve(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	// Reserve 10 small entries — should succeed.
+	entries := make([]ReserveEntry, 10)
+	for i := range entries {
+		entries[i] = ReserveEntry{
+			Key:   []byte("k"),
+			Value: []byte("v"),
+		}
+	}
+	total, err := mt.Reserve(entries)
+	if err != nil {
+		t.Fatalf("Reserve 10 entries: %v", err)
+	}
+	if total == 0 {
+		t.Fatal("expected non-zero total size")
+	}
+
+	// Reserve more than remaining — should fail.
+	big := []ReserveEntry{{
+		Key:   make([]byte, 2048),
+		Value: make([]byte, 2048),
+	}}
+	_, err = mt.Reserve(big)
+	if err == nil {
+		t.Fatal("expected ErrMemTableFull for oversized reserve")
+	}
+}
+
+func TestMemTableConcurrentReads(t *testing.T) {
+	mt := NewMemTable(8192)
+
+	// Write 10 pending entries.
+	for i := 0; i < 10; i++ {
+		key := []byte{byte('a' + i)}
+		val := []byte{byte(i)}
+		if err := mt.PutPending(key, val, uint64(i+1)); err != nil {
+			t.Fatalf("PutPending %d: %v", i, err)
+		}
+	}
+
+	// Publish all.
+	mt.Publish(10)
+
+	// Concurrent reads should all see published values.
+	var wg sync.WaitGroup
+	for g := 0; g < 4; g++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				key := []byte{byte('a' + (i % 10))}
+				res := mt.Get(key)
+				if !res.Found {
+					t.Errorf("key %s not found", key)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestMemTableIteratorSkipsPending(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	_ = mt.PutPending([]byte("pending"), []byte("p"), 1)
+	_ = mt.PutPending([]byte("published"), []byte("pub"), 2)
+	mt.Publish(2) // publishes "published" at seq 2
+
+	// Abort seq 1 — stays pending.
+	// Actually we already published up to 2 which would publish seq 1 too.
+	// Let's test differently: insert a pending entry after publish.
+	_ = mt.PutPending([]byte("still_pending"), []byte("sp"), 3)
+
+	it := mt.NewIterator()
+	count := 0
+	for it.Valid() {
+		count++
+		it.Next()
+	}
+
+	// Should see "pending" (seq 1, now published) and "published" (seq 2),
+	// but not "still_pending" (seq 3, still pending).
+	if count != 2 {
+		t.Fatalf("expected 2 visible entries, got %d", count)
+	}
+}
+
+func TestMemTableDeletePending(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	// Insert then delete.
+	_ = mt.PutPending([]byte("key1"), []byte("val1"), 1)
+	_ = mt.DeletePending([]byte("key1"), 2)
+
+	mt.Publish(2)
+
+	// After publish, the latest entry is a tombstone (nil value).
+	res := mt.Get([]byte("key1"))
+	if !res.Found {
+		t.Fatal("tombstone entry should still be 'found'")
+	}
+	if res.Value != nil {
+		t.Fatalf("expected nil value for tombstone, got %q", res.Value)
+	}
+}
+
+func TestMemTableApproximateSize(t *testing.T) {
+	mt := NewMemTable(4096)
+
+	if mt.ApproximateSize() != 0 {
+		t.Fatal("new memtable should have zero approximate size")
+	}
+	if mt.UsableCapacity() != 4096 {
+		t.Fatalf("usable capacity: got %d, want 4096", mt.UsableCapacity())
+	}
+}