From cf913b1d5299da6f3d2a54f58b918231dc083bae Mon Sep 17 00:00:00 2001 From: dailz Date: Fri, 12 Jun 2026 13:23:27 +0800 Subject: [PATCH] feat: initialize project structure and error types - go.mod with github.com/dailz/go-kv, Go 1.26.3, testify - config/config.go with WalConfig, Validate() with checked arithmetic - errors.go with sentinel errors (ErrCommitUnknown, ErrWriteStopped, etc.) - wal/constants.go with all WAL format constants and enums - wal/header.go with WAL File Header encode/decode (CRC32 IEEE) - wal/record.go with Physical Record codec, block boundary, SplitIntoRecords - wal/entry.go with WAL Entry codec (varint keys/values, OpType, ValueKind) - wal/sequence.go with SequenceManager (atomic, CAS, overflow-safe) - manifest/manifest.go with MANIFEST stub (Load/Save atomic) - manifest/current.go with CURRENT file (WriteCurrent/ReadCurrent) - Comprehensive tests for all modules - .golangci.yml configuration --- .golangci.yml | 24 + .omo/boulder.json | 89 + .omo/notepads/phase1-wal/decisions.md | 0 .omo/notepads/phase1-wal/issues.md | 0 .omo/notepads/phase1-wal/learnings.md | 0 .omo/notepads/phase1-wal/problems.md | 0 .omo/plans/phase1-wal.md | 1718 +++++++++++++++++ .../ses_1465a5321ffevd4inJJZXxFenJ.json | 10 + .../ses_1468f0cc5ffew53IA0I4W6HijM.json | 10 + .../ses_1469860f1ffeAxT7jbUD3T1ZK5.json | 10 + .../ses_146adbaa4ffehxje0lmKRg4ZpT.json | 10 + .../ses_146b864adffetMjZkc08p16onz.json | 10 + .../ses_146c330caffehDbiMQ7M0jL3Il.json | 10 + .../ses_149585098ffeVZ53676PRrr2ic.json | 10 + .../ses_14960b98effe7YjeN56yvxsoN3.json | 10 + .../ses_14967e191ffeLfQYyfTQHP5TlH.json | 10 + .../ses_149a13a61ffehpTGChldqMe0do.json | 10 + .../ses_149e4f0c5ffe5hf5hErOGwmaO7.json | 10 + .../ses_149f503d8ffeCitCeNNhuI5uPD.json | 10 + .../ses_1551b5cb8ffeRq3O3tTE6gADNa.json | 10 + .../ses_15535520fffenpCk4v55tlxfPN.json | 10 + .../ses_155c694ebffeUxfvz2X0kcpMQz.json | 10 + .../ses_162be27e5ffePD7UoGttrWFm7T.json | 10 + .../ses_16f5a4050ffe4VOJm8EOoPcnZV.json | 10 + .../ses_16f63a221ffew3fp5TM11NaRDD.json | 10 + .../ses_17337fda2fferaZJ6tpXI2Cw44.json | 10 + .../ses_1738cec10ffex8teaUh336WrCD.json | 10 + config/config.go | 205 ++ config/config_test.go | 32 + docs/.markdownlint.json | 7 + docs/issues/oracle-wal-3.2-review.md | 297 +++ docs/phase1-wal-plan.md | 640 ++++++ errors.go | 22 + errors_test.go | 42 + go.mod | 11 + go.sum | 10 + manifest/current.go | 43 + manifest/doc.go | 2 + manifest/manifest.go | 52 + manifest/manifest_test.go | 45 + memtable/doc.go | 2 + wal/constants.go | 80 + wal/constants_test.go | 72 + wal/doc.go | 2 + wal/entry.go | 134 ++ wal/entry_test.go | 114 ++ wal/header.go | 98 + wal/header_test.go | 87 + wal/record.go | 126 ++ wal/record_test.go | 143 ++ wal/sequence.go | 87 + wal/sequence_test.go | 154 ++ 52 files changed, 4538 insertions(+) create mode 100644 .golangci.yml create mode 100644 .omo/boulder.json create mode 100644 .omo/notepads/phase1-wal/decisions.md create mode 100644 .omo/notepads/phase1-wal/issues.md create mode 100644 .omo/notepads/phase1-wal/learnings.md create mode 100644 .omo/notepads/phase1-wal/problems.md create mode 100644 .omo/plans/phase1-wal.md create mode 100644 .omo/run-continuation/ses_1465a5321ffevd4inJJZXxFenJ.json create mode 100644 .omo/run-continuation/ses_1468f0cc5ffew53IA0I4W6HijM.json create mode 100644 .omo/run-continuation/ses_1469860f1ffeAxT7jbUD3T1ZK5.json create mode 100644 .omo/run-continuation/ses_146adbaa4ffehxje0lmKRg4ZpT.json create mode 100644 .omo/run-continuation/ses_146b864adffetMjZkc08p16onz.json create mode 100644 .omo/run-continuation/ses_146c330caffehDbiMQ7M0jL3Il.json create mode 100644 .omo/run-continuation/ses_149585098ffeVZ53676PRrr2ic.json create mode 100644 .omo/run-continuation/ses_14960b98effe7YjeN56yvxsoN3.json create mode 100644 .omo/run-continuation/ses_14967e191ffeLfQYyfTQHP5TlH.json create mode 100644 .omo/run-continuation/ses_149a13a61ffehpTGChldqMe0do.json create mode 100644 .omo/run-continuation/ses_149e4f0c5ffe5hf5hErOGwmaO7.json create mode 100644 .omo/run-continuation/ses_149f503d8ffeCitCeNNhuI5uPD.json create mode 100644 .omo/run-continuation/ses_1551b5cb8ffeRq3O3tTE6gADNa.json create mode 100644 .omo/run-continuation/ses_15535520fffenpCk4v55tlxfPN.json create mode 100644 .omo/run-continuation/ses_155c694ebffeUxfvz2X0kcpMQz.json create mode 100644 .omo/run-continuation/ses_162be27e5ffePD7UoGttrWFm7T.json create mode 100644 .omo/run-continuation/ses_16f5a4050ffe4VOJm8EOoPcnZV.json create mode 100644 .omo/run-continuation/ses_16f63a221ffew3fp5TM11NaRDD.json create mode 100644 .omo/run-continuation/ses_17337fda2fferaZJ6tpXI2Cw44.json create mode 100644 .omo/run-continuation/ses_1738cec10ffex8teaUh336WrCD.json create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 docs/.markdownlint.json create mode 100644 docs/issues/oracle-wal-3.2-review.md create mode 100644 docs/phase1-wal-plan.md create mode 100644 errors.go create mode 100644 errors_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 manifest/current.go create mode 100644 manifest/doc.go create mode 100644 manifest/manifest.go create mode 100644 manifest/manifest_test.go create mode 100644 memtable/doc.go create mode 100644 wal/constants.go create mode 100644 wal/constants_test.go create mode 100644 wal/doc.go create mode 100644 wal/entry.go create mode 100644 wal/entry_test.go create mode 100644 wal/header.go create mode 100644 wal/header_test.go create mode 100644 wal/record.go create mode 100644 wal/record_test.go create mode 100644 wal/sequence.go create mode 100644 wal/sequence_test.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..d2035d3 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,24 @@ +run: + timeout: 5m + +linters: + enable: + - errcheck + - govet + - staticcheck + - unused + - gosimple + - ineffassign + - typecheck + - misspell + - gofmt + +linters-settings: + errcheck: + check-type-assertions: true + govet: + enable-all: true + +issues: + max-issues-per-linter: 50 + max-same-issues: 5 diff --git a/.omo/boulder.json b/.omo/boulder.json new file mode 100644 index 0000000..1e701d1 --- /dev/null +++ b/.omo/boulder.json @@ -0,0 +1,89 @@ +{ + "schema_version": 2, + "active_work_id": "phase1-wal-3bc18f0c", + "works": { + "phase1-wal-3bc18f0c": { + "work_id": "phase1-wal-3bc18f0c", + "active_plan": "/home/dailz/workspace/src/go-kv/.omo/plans/phase1-wal.md", + "plan_name": "phase1-wal", + "status": "active", + "started_at": "2026-06-12T05:09:32.588Z", + "updated_at": "2026-06-12T05:22:47.301Z", + "session_ids": [ + "opencode:ses_145c3bae9ffeTB2zbsTym0Cev8" + ], + "session_origins": { + "opencode:ses_145c3bae9ffeTB2zbsTym0Cev8": "direct" + }, + "agent": "atlas", + "task_sessions": { + "todo:1": { + "task_key": "todo:1", + "task_label": "1", + "task_title": "Project scaffolding + go.mod + config types", + "session_id": "opencode:ses_145c254a5ffemom68fK0rmB26W", + "agent": "Sisyphus-Junior", + "category": "quick", + "updated_at": "2026-06-12T05:15:14.028Z", + "started_at": "2026-06-12T05:14:15.365Z", + "status": "completed", + "ended_at": "2026-06-12T05:15:14.028Z", + "elapsed_ms": 58663 + }, + "todo:2": { + "task_key": "todo:2", + "task_label": "2", + "task_title": "Error types + constants", + "session_id": "opencode:ses_145bdf583ffe6PSExxk7J357zT", + "agent": "Sisyphus-Junior", + "category": "quick", + "updated_at": "2026-06-12T05:22:47.301Z", + "started_at": "2026-06-12T05:19:53.997Z", + "status": "completed", + "ended_at": "2026-06-12T05:22:47.301Z", + "elapsed_ms": 173304 + } + } + } + }, + "active_plan": "/home/dailz/workspace/src/go-kv/.omo/plans/phase1-wal.md", + "started_at": "2026-06-12T05:09:32.588Z", + "status": "active", + "updated_at": "2026-06-12T05:22:47.301Z", + "session_ids": [ + "opencode:ses_145c3bae9ffeTB2zbsTym0Cev8" + ], + "session_origins": { + "opencode:ses_145c3bae9ffeTB2zbsTym0Cev8": "direct" + }, + "plan_name": "phase1-wal", + "task_sessions": { + "todo:1": { + "task_key": "todo:1", + "task_label": "1", + "task_title": "Project scaffolding + go.mod + config types", + "session_id": "opencode:ses_145c254a5ffemom68fK0rmB26W", + "agent": "Sisyphus-Junior", + "category": "quick", + "updated_at": "2026-06-12T05:15:14.028Z", + "started_at": "2026-06-12T05:14:15.365Z", + "status": "completed", + "ended_at": "2026-06-12T05:15:14.028Z", + "elapsed_ms": 58663 + }, + "todo:2": { + "task_key": "todo:2", + "task_label": "2", + "task_title": "Error types + constants", + "session_id": "opencode:ses_145bdf583ffe6PSExxk7J357zT", + "agent": "Sisyphus-Junior", + "category": "quick", + "updated_at": "2026-06-12T05:22:47.301Z", + "started_at": "2026-06-12T05:19:53.997Z", + "status": "completed", + "ended_at": "2026-06-12T05:22:47.301Z", + "elapsed_ms": 173304 + } + }, + "agent": "atlas" +} \ No newline at end of file diff --git a/.omo/notepads/phase1-wal/decisions.md b/.omo/notepads/phase1-wal/decisions.md new file mode 100644 index 0000000..e69de29 diff --git a/.omo/notepads/phase1-wal/issues.md b/.omo/notepads/phase1-wal/issues.md new file mode 100644 index 0000000..e69de29 diff --git a/.omo/notepads/phase1-wal/learnings.md b/.omo/notepads/phase1-wal/learnings.md new file mode 100644 index 0000000..e69de29 diff --git a/.omo/notepads/phase1-wal/problems.md b/.omo/notepads/phase1-wal/problems.md new file mode 100644 index 0000000..e69de29 diff --git a/.omo/plans/phase1-wal.md b/.omo/plans/phase1-wal.md new file mode 100644 index 0000000..bb4958b --- /dev/null +++ b/.omo/plans/phase1-wal.md @@ -0,0 +1,1718 @@ +# Phase 1: WAL 子系统 — Go KV 存储引擎 + +## TL;DR + +> **Quick Summary**: 基于 `docs/design.md` §3.2 实现 WAL 子系统，配合 MemTable（§3.3）、读路径（§3.4）和最小 MANIFEST，构建可嵌入的单 key autocommit KV 存储引擎。 +> +> **Deliverables**: +> - WAL 写入路径（group commit、sequence 管理、segment 轮转、fsync） +> - WAL 恢复路径（segment 扫描、fragment 重组、batch 校验、尾部截断） +> - MemTable（Arena + SkipList + 原子发布） +> - 嵌入式 API（Open/Close/Put/Delete/Get） +> - MANIFEST 最小 stub + CURRENT 文件 +> - 完整测试套件（单元 + 集成 + 故障注入） +> +> **Estimated Effort**: XL（~30-50 个任务） +> **Parallel Execution**: YES — 5 waves +> **Critical Path**: Task 1 → Task 6 → Task 10 → Task 11 → Task 13 → Task 19 → Task 20/21 → F1-F4 + +--- + +## Context + +### Original Request +用户指向 `docs/design.md` §3.2（WAL），要求制定第一阶段开发方案。 + +### Interview Summary +**Key Discussions**: +- 设计文档是唯一权威来源（1189 行，§3.2 WAL 约 730 行） +- 项目为 greenfield — 无任何 Go 源码 +- Phase 1 范围：WAL + MemTable + 读路径 + MANIFEST stub + 嵌入式 API +- 不含 SSTable / Compaction / Value Log / MVCC / 网络层 + +**Research Findings**: +- 项目完全空白（仅有 README.md、.omo/、docs/） +- 无 go.mod、无依赖 +- 设计文档定义了极其精确的崩溃语义和错误分类 + +### Metis Review +**Identified Gaps** (addressed): +- MANIFEST 完整设计不在文档中 → Phase 1 用最小 stub（仅存 recoverySegmentID） +- MemTable 无 SSTable 可刷 → 冻结为 Immutable 后持有在内存，不刷盘；3×64MB 上限后阻塞写入 +- Periodic/Never sync 策略 → Phase 1 只实现 Always +- Group commit 定时器 → time.AfterFunc + reset +- API surface → Put/Delete/Get/Open/Close + GetDurableSequence + IsWriteStopped +- Graceful shutdown → Close() 等待 pending batch、sync WAL、写 MANIFEST +- 文件锁 → Phase 1 不实现，单进程假设 +- Varint → Go 标准库 encoding/binary.PutUvarint，key/value 上限 4KB 保证 varint ≤ 5 bytes + +--- + +## Work Objectives + +### Core Objective +实现一个功能完整的 WAL 子系统，支持 group commit 写入、崩溃恢复、和单 key autocommit 语义，作为后续 SSTable / MVCC 的基础。 + +### Concrete Deliverables +- `wal/` — WAL 编解码、segment 管理、写入器、恢复器 +- `memtable/` — Arena + SkipList + 原子发布 +- `db.go` — DB 入口（Open/Close/Put/Delete/Get） +- `errors.go` — 公共错误类型（ErrCommitUnknown 等） +- `manifest/` — 最小 MANIFEST stub +- 测试覆盖所有关键路径 + +### Definition of Done +- [ ] `go build ./...` 编译通过 +- [ ] `go test ./... -race` 全部通过，无 data race +- [ ] 写入 N 条数据 → Close → Open → Get 全部正确 +- [ ] WAL 尾部损坏 → 恢复成功，完整 batch 不丢失 +- [ ] ErrCommitUnknown 后引擎进入 write-stopped + +### Must Have +- WAL 写入路径严格遵循设计文档步骤 ①-⑪ +- Per-batch private encode buffer + direct write（无 bufio.Writer） +- `atomic.Uint64` 用于 publishedSequence / durableSequence / nextSequence +- `atomic.Pointer` 用于 skiplist next 指针 +- MemTable Arena 容量预留发生在 WAL write 之前 +- Sequence 分配前完成所有可失败校验 +- WAL Batch 不跨 segment +- Segment rotation 只在 batch 边界 +- `Get` 返回 `GetResult{Found bool, Value []byte}` 区分 key-not-found 与 empty-value +- Config 不变量在 Open 时校验（checked arithmetic） +- Recovery 严格遵循 fragment 状态机 +- 错误分类严格遵循设计文档表格 + +### Must NOT Have (Guardrails) +- ❌ SSTable / Compaction / Bloom Filter / Block Cache +- ❌ Value Log / ValueLogPointer 写入（Entry 类型定义可预留，但 Phase 1 不写入 ValueLogPointer） +- ❌ MVCC / SSI 事务 +- ❌ Periodic / Never sync 策略（仅 Always） +- ❌ Range scan / Iterator API +- ❌ 网络层 +- ❌ `bufio.Writer` 在 WAL append 路径上 +- ❌ MemTable 写入失败发生在 WAL write 成功之后 +- ❌ CRC 校验跳过 +- ❌ 共享 buffered writer 模糊 WAL 副作用边界 + +--- + +## Verification Strategy + +> **ZERO HUMAN INTERVENTION** — ALL verification is agent-executed. No exceptions. + +### Test Decision +- **Infrastructure exists**: NO (greenfield) +- **Automated tests**: Tests-after（先实现后测试，每个模块实现完即写测试） +- **Framework**: Go 标准 `testing` + `testify`（断言库） +- **Agent-Executed QA**: 每个任务包含 `go test` 命令和 `go test -race` 验证 + +### QA Policy +- 每个 task 的 QA 通过 `go test` 和 `go vet` 执行 +- Evidence: test output saved to `.omo/evidence/task-{N}-{scenario}.txt` + +--- + +## Execution Strategy + +### Parallel Execution Waves + +``` +Wave 1a (Foundation — 1 task): +└── 1. Project scaffolding + go.mod + config types [quick] + +Wave 1b (Codec + Types — 4 parallel + 2 independent, after Wave 1a): +├── 2. Error types + constants [quick] +├── 3. WAL File Header encode/decode [quick] +├── 4. Physical Record encode/decode + Block boundary [quick] +├── 5. WAL Entry encode/decode [quick] +├── 12. Sequence manager [quick] +└── 18. MANIFEST stub + CURRENT file [quick] + +Wave 2a (Batch + Arena + Validation — 3 parallel, after Wave 1b): +├── 6. WAL Batch encode/decode + fragment collector [unspecified-high] (needs 3,4,5) +├── 7. Arena allocator [unspecified-high] (needs 1) +└── 9. WAL Batch resource validation [quick] (needs 1, 5) + +Wave 2b (SkipList — after Arena): +└── 8. SkipList (mutex write + lock-free read) [deep] (needs 7) + +Wave 3a (Segment writer — after Batch codec): +└── 10. Segment writer (file write + block writer) [unspecified-high] (needs 3,4,6) + +Wave 3b (Segment rotation — after Segment writer): +└── 11. Segment rotation + durable-ready protocol [unspecified-high] (needs 10) + +Wave 3c (MemTable — after Arena + SkipList): +└── 14. MemTable (wrap skiplist + arena + publish/abort) [unspecified-high] (needs 7,8) + +Wave 3d (WAL Writer — after MemTable API available): +└── 13. Commit queue + WAL writer main loop [deep] (needs 2,9,10,11,12,14) + +Wave 4a (Recovery scanner — after Segment writer): +└── 15. WAL recovery: scanner + record parser [unspecified-high] (needs 4,10) + +Wave 4b (Recovery batch replay — after scanner + Batch codec): +└── 16. WAL recovery: fragment collector + batch replay [deep] (needs 6,15) + +Wave 4c (Recovery tail truncation — after batch replay): +└── 17. WAL recovery: tail truncation + main flow [unspecified-high] (needs 15,16) + +Wave 4d (DB integration — after all above): +└── 19. DB integration: Open/Close/Put/Delete/Get [deep] (needs 13,14,17,18) + +Wave 5 (Tests + Benchmark — after DB integration): +├── 20. End-to-end + crash recovery tests [deep] +└── 21. Benchmark suite [unspecified-high] + +Wave FINAL (4 parallel reviews): +├── F1. Plan compliance audit (oracle) +├── F2. Code quality review (unspecified-high) +├── F3. Real manual QA (unspecified-high) +└── F4. Scope fidelity check (deep) + +Critical Path: 1 → 6 → 10 → 11 → 13 → 19 → 20/21 → F1-F4 +Parallel Speedup: ~50% faster than sequential +Max Concurrent: 6 (Wave 1b) +``` + +### Dependency Matrix + +| Task | Depends On | Blocks | Sub-Wave | +|------|-----------|--------|----------| +| 1 | — | 2-5, 7, 9, 12, 18 | 1a | +| 2 | 1 | 13 | 1b | +| 3 | 1 | 6, 10 | 1b | +| 4 | 1 | 6, 10, 15 | 1b | +| 5 | 1 | 6, 9 | 1b | +| 6 | 3, 4, 5 | 10, 16 | 2a | +| 7 | 1 | 8, 14 | 2a | +| 8 | 7 | 14 | 2b | +| 9 | 1, 5 | 13 | 2a | +| 10 | 3, 4, 6 | 11, 15 | 3a | +| 11 | 10 | 13 | 3b | +| 12 | 1 | 13 | 1b | +| 13 | 2, 9, 10, 11, 12, 14 | 19 | 3d | +| 14 | 7, 8 | 19 | 3c | +| 15 | 4, 10 | 16 | 4a | +| 16 | 6, 15 | 17 | 4b | +| 17 | 15, 16 | 19 | 4c | +| 18 | 1 | 19 | 1b | +| 19 | 13, 14, 17, 18 | 20, 21 | 4d | +| 20 | 19 | F1-F4 | 5 | +| 21 | 19 | F1-F4 | 5 | + +### Agent Dispatch Summary + +- **1a**: **1** — T1 → `quick` +- **1b**: **6** — T2 → `quick`, T3 → `quick`, T4 → `quick`, T5 → `quick`, T12 → `quick`, T18 → `quick` +- **2a**: **3** — T6 → `unspecified-high`, T7 → `unspecified-high`, T9 → `quick` +- **2b**: **1** — T8 → `deep` +- **3a**: **1** — T10 → `unspecified-high` +- **3b**: **1** — T11 → `unspecified-high` +- **3c**: **1** — T14 → `unspecified-high` +- **3d**: **1** — T13 → `deep` +- **4a**: **1** — T15 → `unspecified-high` +- **4b**: **1** — T16 → `deep` +- **4c**: **1** — T17 → `unspecified-high` +- **4d**: **1** — T19 → `deep` +- **5**: **2** — T20 → `deep`, T21 → `unspecified-high` +- **FINAL**: **4** — F1 → `oracle`, F2 → `unspecified-high`, F3 → `unspecified-high`, F4 → `deep` + +--- + +## TODOs + +- [x] 1. Project scaffolding + go.mod + config types + + **What to do**: + - 初始化 `go.mod`（模块名 `github.com/dailz/go-kv`，Go 1.22+） + - 创建目录结构：`wal/`、`memtable/`、`manifest/`、`config/` + - 创建 `config/config.go`：`WalConfig` 结构体（MaxSegmentSize, BlockSize, SyncMode, MaxBatchEntries, MaxBatchSize, MaxKeyBytes, MaxInlineValue, MemTableSize, MaxImmutableCount） + - 创建 `config/config.go`：`Validate() error` 函数，用 checked arithmetic 校验不变量：`maxWalSegmentPayload >= maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding` + - 计算公式参照设计文档 lines 402-453：walFileHeaderSize=32, walBlockSize=32768, physicalRecordHeaderSize=7, walBatchHeaderSize=18, maxWalBatchEntriesSize=4MB + - 创建 `.golangci.yml` + + **Must NOT do**: + - 不引入外部依赖（仅标准库 + testify） + - 不定义 SSTable / Compaction 相关配置 + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [`golang-project-layout`] + - `golang-project-layout`: Go 项目结构最佳实践 + - **Skills Evaluated but Omitted**: + - `golang-cli`: 不是 CLI 工具 + + **Parallelization**: + - **Can Run In Parallel**: NO (foundation task) + - **Parallel Group**: Sub-Wave 1a (solo — blocks all others) + - **Blocks**: 2, 3, 4, 5, 7, 9, 12, 18 + - **Blocked By**: None + + **References**: + **Pattern References**: + - `docs/design.md:286-289` — 设计决策表：批量窗口、日志格式、文件管理、segment 大小 + - `docs/design.md:402-453` — 配置不变量计算公式和最小 segment 大小推导 + + **API/Type References**: + - `docs/design.md:501-513` — WAL Batch 资源上限默认值 + + **External References**: + - Go project layout: https://go.dev/doc/modules/layout + + **WHY Each Reference Matters**: + - 设计决策表定义了所有默认值，config 结构体必须映射这些值 + - 配置不变量公式是 Open 时必须校验的核心约束 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Config validation accepts valid defaults + Tool: Bash + Preconditions: Go toolchain installed + Steps: + 1. cd /home/dailz/workspace/src/go-kv + 2. Create a test file config/config_test.go with TestValidateDefaults + 3. Test that WalConfig with default values passes Validate() + Expected Result: go test ./config/... passes + Evidence: .omo/evidence/task-1-config-defaults.txt + + Scenario: Config validation rejects invalid segment size + Tool: Bash + Preconditions: go.mod exists + Steps: + 1. Create TestValidateSegmentTooSmall in config/config_test.go + 2. Set MaxSegmentSize = 1024 (far below minimum 4,195,264) + 3. Verify Validate() returns error containing "segment" + Expected Result: Test fails validation with descriptive error + Evidence: .omo/evidence/task-1-config-invalid.txt + ``` + + **Commit**: YES (group with Task 2) + - Message: `feat: initialize project structure and error types` + - Files: `go.mod, config/, .golangci.yml` + +- [x] 2. Error types + constants + + **What to do**: + - 创建 `errors.go`：定义 `ErrCommitUnknown`、`ErrWriteStopped`、`ErrSequenceExhausted`、`ErrWALCorrupted`、`ErrInvalidConfig` + - 创建 `wal/constants.go`：所有 WAL 常量（WalMagic, WalFormatVersion, WalFileHeaderSize, WalBlockSize, PhysicalRecordHeaderSize, WalBatchHeaderSize, MaxWalBatchEntryCount, MaxWalBatchEntriesSize, MaxWalKeyBytes, MaxWalInlineValueBytes, MaxWalVarintBytes, DefaultMaxWalSegmentSize） + - 创建 `wal/constants.go`：Fragment types (RecInvalid=0, RecFull=1, RecFirst=2, RecMiddle=3, RecLast=4) + - 创建 `wal/constants.go`：OpType (OpInvalid=0, OpPut=1, OpDelete=2) 和 ValueKind (VKNone=0, VKInline=1, VKValueLogPointer=2) + + **Must NOT do**: + - 不实现具体逻辑，仅类型和常量定义 + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + - **Skills Evaluated but Omitted**: + - `golang-naming`: 常量命名简单，不需要完整命名 skill + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 1b (with Tasks 3, 4, 5, 12, 18) + - **Blocks**: 13 (WAL writer references error types) + - **Blocked By**: 1 (needs go.mod) + + **References**: + **Pattern References**: + - `docs/design.md:129-135` — 错误分类表（5 种失败场景 + ErrCommitUnknown） + - `docs/design.md:155` — ErrCommitUnknown 的 maybe committed 语义 + - `docs/design.md:496-499` — sequence exhausted 终态 + + **API/Type References**: + - `docs/design.md:501-513` — 所有资源上限默认值 + - `docs/design.md:364-373` — Fragment type 枚举 + - `docs/design.md:571-577` — OpType 枚举 + - `docs/design.md:580-584` — ValueKind 枚举 + + **WHY Each Reference Matters**: + - 错误类型直接映射设计文档的错误分类表 + - 常量值必须与设计文档精确匹配（block size=32KB, header=32 bytes 等） + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Error types are distinguishable via errors.Is + Tool: Bash + Preconditions: Task 1 complete + Steps: + 1. Create errors_test.go with TestErrorTypes + 2. Verify errors.Is(ErrCommitUnknown, ErrCommitUnknown) == true + 3. Verify errors.Is(ErrWriteStopped, ErrCommitUnknown) == false + Expected Result: All error type identity checks pass + Evidence: .omo/evidence/task-2-error-types.txt + + Scenario: Constants match design doc values + Tool: Bash + Preconditions: wal/constants.go exists + Steps: + 1. Create wal/constants_test.go with TestConstantValues + 2. Assert WalBlockSize == 32*1024 + 3. Assert WalFileHeaderSize == 32 + 4. Assert MaxWalBatchEntriesSize == 4*1024*1024 + 5. Assert DefaultMaxWalSegmentSize == 64*1024*1024 + Expected Result: All assertions pass + Evidence: .omo/evidence/task-2-constants.txt + ``` + + **Commit**: YES (group with Task 1) + - Message: `feat: initialize project structure and error types` + - Files: `errors.go, wal/constants.go` + +- [x] 3. WAL File Header encode/decode + + **What to do**: + - 创建 `wal/header.go`：`WalFileHeader` 结构体（Magic uint32, FormatVersion uint16, HeaderSize uint16, BlockSize uint32, SegmentID uint64, StartSequence uint64, HeaderCRC uint32） + - `EncodeWalHeader(h *WalFileHeader) [WalFileHeaderSize]byte` — 编码为 little-endian 32 bytes，headerCRC 覆盖 magic 到 startSequence（不包含自身） + - `DecodeWalHeader(data []byte) (*WalFileHeader, error)` — 校验 magic、formatVersion、headerSize、headerCRC + - CRC 使用 `hash/crc32` with IEEE table（或 CRC32C 如果标准库支持） + - 创建 table-driven tests + + **Must NOT do**: + - 不使用外部 CRC 库 + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 1b (with Tasks 2, 4, 5, 12, 18) + - **Blocks**: 6 (batch codec), 10 (segment writer) + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:316-341` — WAL File Header 格式定义和字段说明 + + **WHY Each Reference Matters**: + - Header 格式是 WAL segment 文件的基础，所有后续 segment 操作都依赖正确编码 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Header encode/decode round-trip + Tool: Bash + Preconditions: wal/header.go exists + Steps: + 1. Create TestHeaderRoundtrip in wal/header_test.go + 2. Encode a header with SegmentID=5, StartSequence=1000 + 3. Decode the encoded bytes + 4. Assert all fields match original + Expected Result: Round-trip preserves all fields + Evidence: .omo/evidence/task-3-header-roundtrip.txt + + Scenario: Header CRC detects corruption + Tool: Bash + Steps: + 1. Encode a valid header + 2. Flip one byte in the magic field + 3. Attempt DecodeWalHeader + 4. Assert error returned containing "crc" or "checksum" + Expected Result: Corrupted header rejected + Evidence: .omo/evidence/task-3-header-crc.txt + ``` + + **Commit**: YES (group with Tasks 4, 5) + - Message: `feat(wal): implement WAL entry/record/header codec` + +- [x] 4. Physical Record encode/decode + Block boundary + + **What to do**: + - 创建 `wal/record.go`：`PhysicalRecord` 结构体（CRC uint32, Length uint16, Type uint8, Payload []byte） + - `EncodePhysicalRecord(recType uint8, payload []byte) []byte` — 编码为 7-byte header + payload + - `DecodePhysicalRecord(data []byte) (rec *PhysicalRecord, consumed int, err error)` — 解析并校验 CRC + - CRC 覆盖 length + type + payload（不含 CRC 自身） + - Block 边界辅助函数： + - `PaddingNeeded(blockOffset uint32) int` — 剩余 <= 7 bytes 时返回 padding 大小 + - `CanFitRecord(blockOffset uint32, payloadLen uint32) bool` + - `SplitIntoRecords(encodedBatch []byte) [][]byte` — 将编码后的 batch 按 block 边界拆分为 Physical Record payloads + - Table-driven tests 覆盖：单 record、跨 block 拆分、padding + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 1b (with Tasks 2, 3, 5, 12, 18) + - **Blocks**: 6, 10, 15 + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:345-381` — Physical Record 格式、fragment 类型、合法组合 + - `docs/design.md:383-391` — Block 边界处理规则 + + **WHY Each Reference Matters**: + - Physical Record 是 WAL I/O 的最小单元，block 边界处理是拆分/重组的关键 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Physical Record round-trip + Tool: Bash + Steps: + 1. Create TestRecordRoundtrip in wal/record_test.go + 2. Encode RecFull with payload "hello world" + 3. Decode the encoded bytes + 4. Assert Type==RecFull, Payload=="hello world" + Expected Result: Round-trip correct + Evidence: .omo/evidence/task-4-record-roundtrip.txt + + Scenario: Batch split into fragments across blocks + Tool: Bash + Steps: + 1. Create TestSplitIntoRecords in wal/record_test.go + 2. Create a 40KB payload (larger than 32KB block) + 3. Call SplitIntoRecords with blockSize=32KB + 4. Assert result has First + Last (or First + Middle + Last) + 5. Concatenate all payloads → equals original + Expected Result: Split produces valid fragment sequence, reassembly equals original + Evidence: .omo/evidence/task-4-split-records.txt + + Scenario: Block padding when remaining <= 7 bytes + Tool: Bash + Steps: + 1. Create TestBlockPadding in wal/record_test.go + 2. blockOffset = 32*1024 - 5 (only 5 bytes remaining) + 3. Assert PaddingNeeded(blockOffset) == 5 + 4. Assert CanFitRecord(blockOffset, 1) == false + Expected Result: Padding and fit check correct + Evidence: .omo/evidence/task-4-padding.txt + ``` + + **Commit**: YES (group with Tasks 3, 5) + +- [x] 5. WAL Entry encode/decode + + **What to do**: + - 创建 `wal/entry.go`：`WalEntry` 结构体（OpType uint8, ValueKind uint8, Key []byte, Value []byte） + - `EncodeEntry(e *WalEntry) ([]byte, error)` — 编码 opType(u8) + valueKind(u8) + keyLen(varint) + valLen(varint) + key + value + - `DecodeEntry(data []byte) (entry *WalEntry, consumed int, err error)` — 解码返回 entry 和 consumed bytes + - Varint 使用 `encoding/binary.PutUvarint` / `binary.ReadUvarint` + - 校验规则（参照 design.md lines 586-594）： + - keyLen > 0 && keyLen <= MaxWalKeyBytes + - Put 要求 valueKind ∈ {VKInline, VKValueLogPointer} + - Put + VKInline: valLen <= MaxWalInlineValueBytes (允许 valLen = 0) + - Put + VKValueLogPointer: valLen > 0 + - Delete: valueKind == VKNone, valLen == 0 + - Table-driven tests 覆盖所有合法/非法组合 + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 1b (with Tasks 2, 3, 4, 12, 18) + - **Blocks**: 6, 9 + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:558-594` — Entry 格式、OpType、ValueKind、合法性规则 + + **WHY Each Reference Matters**: + - Entry 是 WAL 的最小逻辑单元，编解码正确性直接影响数据完整性 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Entry encode/decode round-trip for all valid types + Tool: Bash + Steps: + 1. Create TestEntryRoundtrip in wal/entry_test.go + 2. Test cases: Put+Inline("key1","val1"), Put+Inline("key2",[]byte{}), Delete("key3") + 3. Encode → Decode → assert all fields match + Expected Result: All round-trips correct + Evidence: .omo/evidence/task-5-entry-roundtrip.txt + + Scenario: Entry validation rejects invalid entries + Tool: Bash + Steps: + 1. Create TestEntryValidation in wal/entry_test.go + 2. Test: OpInvalid → error + 3. Test: Put+VKNone → error + 4. Test: Delete+VKInline → error + 5. Test: keyLen=0 → error + 6. Test: keyLen > MaxWalKeyBytes → error + Expected Result: All invalid entries rejected with descriptive errors + Evidence: .omo/evidence/task-5-entry-validation.txt + ``` + + **Commit**: YES (group with Tasks 3, 4) + +- [ ] 6. WAL Batch encode/decode + fragment collector + + **What to do**: + - 创建 `wal/batch.go`：`WalBatch` 结构体（Flags uint16, BaseSequence uint64, EntryCount uint32, EntriesSize uint32, Entries []byte） + - `EncodeWalBatch(baseSequence uint64, entries []*WalEntry) ([]byte, error)` — 编码 Batch Header + 序列化 Entries + - `DecodeWalBatch(data []byte) (*WalBatch, error)` — 校验 flags、entryCount（0 < N <= max）、entriesSize（与实际 bytes 匹配） + - `FragmentCollector` 结构体 — 实现 Idle / CollectingFragments 状态机 + - `Reset()` + - `Append(recType uint8, payload []byte) error` — 按 fragment type 追加，校验 buffer 大小上限 + - `IsComplete() bool` — 是否拼出完整 batch + - `BatchData() []byte` — 返回重组后的完整 batch bytes + - `State() FragmentState` — 当前状态（Idle / CollectingFragments） + - 状态转移严格遵循 design.md lines 706-731 的状态机 + - Buffer 大小限制：Batch Header 长度 + MaxWalBatchEntriesSize + - Varint 长度限制：MaxWalVarintBytes = 5 + + **Must NOT do**: + - FragmentCollector 不做 CRC 校验（由 Physical Record 层负责） + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES (with Tasks 7, 9 — all depend only on Wave 1b outputs) + - **Parallel Group**: Sub-Wave 2a (with Tasks 7, 9) + - **Blocks**: 10, 16 + - **Blocked By**: 3, 4, 5 + + **References**: + **Pattern References**: + - `docs/design.md:455-530` — WAL Batch 格式、Batch Header、Entry sequence 推导 + - `docs/design.md:706-731` — Fragment 状态机定义 + + **WHY Each Reference Matters**: + - Batch 是 WAL 的核心持久化单元，FragmentCollector 是 recovery 的核心组件 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Batch encode/decode round-trip with multiple entries + Tool: Bash + Steps: + 1. Create TestBatchRoundtrip in wal/batch_test.go + 2. Create 10 entries: 5 Put+Inline, 3 Delete, 2 Put+Inline with empty value + 3. Encode with baseSequence=42 + 4. Decode → assert Flags, BaseSequence, EntryCount, EntriesSize match + 5. Parse all entries from Entries bytes → assert each matches original + Expected Result: Full round-trip preserves all data + Evidence: .omo/evidence/task-6-batch-roundtrip.txt + + Scenario: Fragment collector state machine + Tool: Bash + Steps: + 1. Create TestFragmentCollector in wal/batch_test.go + 2. Encode a 100KB batch, split into records (First + Middle* + Last) + 3. Feed fragments to collector one by one + 4. Assert state transitions: Idle → CollectingFragments → ... → Idle + 5. Assert collected batch bytes equal original encoded batch + Expected Result: Fragment reassembly produces original batch + Evidence: .omo/evidence/task-6-fragment-collector.txt + + Scenario: Fragment collector rejects illegal transitions + Tool: Bash + Steps: + 1. Feed RecMiddle to Idle state → expect error + 2. Feed RecLast to Idle state → expect error + 3. Feed RecFull to CollectingFragments state → expect error + 4. Feed RecFirst to CollectingFragments state → expect error + Expected Result: All illegal transitions rejected + Evidence: .omo/evidence/task-6-fragment-illegal.txt + ``` + + **Commit**: YES (group with Task 9) + - Message: `feat(wal): implement WAL batch codec and resource validation` + +- [ ] 7. Arena allocator + + **What to do**: + - 创建 `memtable/arena.go`：`Arena` 结构体 + - 固定大小 byte slice（默认 64MB，由 config.MemTableSize 配置） + - `Allocate(size uint32) (offset uint32, err error)` — 分配对齐的内存块，返回 offset；满时返回 error + - `GetBytes(offset uint32, size uint32) []byte` — 从 offset 获取 bytes slice + - `Remaining() uint32` — 剩余可用 bytes + - `Capacity() uint32` — 总容量 + - `Reserve(totalSize uint32) error` — 检查剩余容量是否足够（不实际分配） + - 对齐：8-byte 对齐分配 + - 线程安全：分配使用 sync.Mutex（写路径已被 WAL writer 串行化） + - 测试：分配、对齐、满时错误、并发安全 + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [`golang-concurrency`] + - `golang-concurrency`: Arena 需要线程安全分配 + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 2a (with Tasks 6, 9 — all depend only on Task 1 or Wave 1b outputs) + - **Blocks**: 8, 14 + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:864-879` — Arena 容量预留规则 + + **WHY Each Reference Matters**: + - Arena 是 MemTable 的内存基础，容量预留是 WAL 写入路径的前置约束 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Arena allocate and read back + Tool: Bash + Steps: + 1. Create TestArenaAllocate in memtable/arena_test.go + 2. Create Arena with 1024 bytes + 3. Allocate 100 bytes → write "test data" at offset 0 + 4. GetBytes(offset, 9) → assert equals "test data" + 5. Allocate another 200 bytes → assert offset is 8-byte aligned + Expected Result: Allocation and readback correct, alignment enforced + Evidence: .omo/evidence/task-7-arena-alloc.txt + + Scenario: Arena full returns error + Tool: Bash + Steps: + 1. Create Arena with 64 bytes + 2. Allocate 60 bytes → success + 3. Allocate 10 bytes → error (not enough remaining) + Expected Result: Full arena rejects allocation + Evidence: .omo/evidence/task-7-arena-full.txt + ``` + + **Commit**: YES (with Task 8) + - Message: `feat(memtable): implement Arena allocator and SkipList` + +- [ ] 8. SkipList (mutex write + lock-free read) + + **What to do**: + - 创建 `memtable/skiplist.go`：`SkipList` 结构体 + - 最大 20 层，p=0.25 的概率晋升 + - `Put(key []byte, value []byte, sequence uint64, nodeData interface{}) error` — Mutex 保护下的有序插入 + - `Get(key []byte) (found bool, value []byte, sequence uint64)` — 无锁读 + - `NewIterator() *Iterator` — 有序遍历 + - **原子发布**：所有 next 指针使用 `atomic.Pointer[skipNode]`，store 使用 release 语义 + - 节点包含：key、value、sequence、pending 标记（用 atomic.Bool 或存入 nodeData） + - Key 比较使用 bytes.Compare + - Arena 分配：节点在 Arena 中分配（offset-based 引用，通过 Arena.GetBytes 访问） + - 测试：有序插入、查找、遍历、并发读写（`-race`） + + **Must NOT do**: + - 不使用普通指针（必须 atomic.Pointer） + - 不依赖 Mutex 向读者发布内存（必须 atomic store-release） + + **Recommended Agent Profile**: + - **Category**: `deep` + - **Skills**: [`golang-concurrency`] + - `golang-concurrency`: 原子操作和内存序是核心难点 + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on Task 7) + - **Parallel Group**: Sub-Wave 2b (solo — after Arena complete) + - **Blocks**: 14 + - **Blocked By**: 7 + + **References**: + **Pattern References**: + - `docs/design.md:834-840` — SkipList 设计决策（20 层、64MB Arena、Mutex 写 + 无锁读） + - `docs/design.md:841-850` — 发布与内存序约束（atomic.Pointer, release store） + + **WHY Each Reference Matters**: + - 内存序约束是整个读路径正确性的关键，ARM 等弱内存序平台必须严格遵守 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: SkipList ordered insertion and retrieval + Tool: Bash + Steps: + 1. Create TestSkipListOrdered in memtable/skiplist_test.go + 2. Insert keys "c", "a", "e", "b", "d" with values "val-C" etc. + 3. Get each key → assert correct value + 4. Iterate → assert order is a, b, c, d, e + Expected Result: Keys stored and retrieved in correct order + Evidence: .omo/evidence/task-8-skiplist-ordered.txt + + Scenario: SkipList concurrent read/write no data race + Tool: Bash + Steps: + 1. Create TestSkipListConcurrent in memtable/skiplist_test.go + 2. Start 4 writer goroutines inserting 100 keys each + 3. Start 4 reader goroutines reading random keys + 4. Run with go test -race + Expected Result: No data race detected + Evidence: .omo/evidence/task-8-skiplist-race.txt + ``` + + **Commit**: YES (with Task 7) + +- [ ] 9. WAL Batch resource validation + + **What to do**: + - 创建 `wal/validate.go`：`ValidateBatchLimits(entries []*WalEntry, cfg *config.WalConfig) error` + - 在 sequence 分配之前检查所有可失败条件： + - entryCount > 0 && entryCount <= cfg.MaxBatchEntries + - 每个 entry 的 keyLen > 0 && keyLen <= cfg.MaxKeyBytes + - 每个 entry 如果是 Put+Inline 则 valLen <= cfg.MaxInlineValue + - 所有 entries 编码后总大小 <= cfg.MaxBatchSize + - worstCase Physical Record overhead + batch encoded size <= maxWalSegmentPayload + - checked arithmetic：计算中不溢出 + - 返回的具体错误类型允许区分不同违规 + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Sub-Wave 2a (with Tasks 6, 7 — all depend on Task 1/Wave 1b) + - **Blocks**: 13 + - **Blocked By**: 1, 5 + + **References**: + **Pattern References**: + - `docs/design.md:501-513` — WAL Batch 资源上限 + - `docs/design.md:514` — "写入侧必须在 sequence 分配、MemTable 预留和 WAL append 之前完成这些校验" + + **WHY Each Reference Matters**: + - 这些校验必须在 sequence 分配之前完成，否则失败会触发 write-stopped + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Valid batch passes validation + Tool: Bash + Steps: + 1. Create TestValidateBatchLimits in wal/validate_test.go + 2. Create 100 Put+Inline entries with 10-byte keys and 10-byte values + 3. Call ValidateBatchLimits → expect nil error + Expected Result: Valid batch accepted + Evidence: .omo/evidence/task-9-validate-pass.txt + + Scenario: Batch exceeding entry count limit rejected + Tool: Bash + Steps: + 1. Create 10,001 entries (exceeds default MaxBatchEntries=10000) + 2. Call ValidateBatchLimits → expect error containing "entry count" + Expected Result: Over-limit batch rejected before sequence allocation + Evidence: .omo/evidence/task-9-validate-entrycount.txt + + Scenario: Key exceeding size limit rejected + Tool: Bash + Steps: + 1. Create entry with keyLen = 5KB (exceeds MaxWalKeyBytes=4KB) + 2. Call ValidateBatchLimits → expect error containing "key" + Expected Result: Oversized key rejected + Evidence: .omo/evidence/task-9-validate-keysize.txt + ``` + + **Commit**: YES (group with Task 6) + +- [ ] 10. Segment writer (file write + block writer) + + **What to do**: + - 创建 `wal/block_writer.go`：`BlockWriter` — 管理 32KB block 的 Physical Record 写入 + - 内部维护当前 block 的 byte buffer + - 写入 Physical Record 时检查 block 边界 + - 剩余 <= 7 bytes 时自动 padding（全 0） + - 跨 block 的 batch fragment 自动拆分 + - `WriteRecord(recType uint8, payload []byte) error` + - `Flush(io.Writer) error` — 将当前 block 写入 fd + - `Reset()` — 重置为空 block + - 创建 `wal/segment_writer.go`：`SegmentWriter` + - 封装 WAL segment 文件的追加写入 + - `NewSegmentWriter(dir string, segmentID uint64, startSequence uint64, cfg *config.WalConfig) (*SegmentWriter, error)` — 创建 segment-N.wal.tmp → 写 header → fsync → rename → fsync dir → durable-ready + - `AppendBatch(encodedBatch []byte) error` — 按 block 边界写入 batch（使用 BlockWriter） + - `Sync() error` — fsync 当前文件 + - `Close() error` + - `RemainingPayload() uint64` — 当前 segment 剩余 payload + - `CurrentOffset() uint64` + - `SegmentID() uint64` + - `EndState() SegmentEndState` — 返回 (segmentID, endOffset, endSequence) + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on 3, 4, 6) + - **Parallel Group**: Sub-Wave 3a (solo — after batch codec complete) + - **Blocks**: 11, 15 + - **Blocked By**: 3, 4, 6 + + **References**: + **Pattern References**: + - `docs/design.md:247-278` — WAL 元数据持久化协议（durable-ready） + - `docs/design.md:383-391` — Block 边界处理规则 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Segment creation follows durable-ready protocol + Tool: Bash + Steps: + 1. Create TestSegmentCreation in wal/segment_writer_test.go + 2. Create a temp directory, create SegmentWriter with segmentID=0 + 3. Verify segment-0.wal exists in directory + 4. Read file → verify header is correct (magic, segmentID, startSequence) + 5. Verify no .tmp file remains + Expected Result: Segment file created with correct header, no temp file + Evidence: .omo/evidence/task-10-segment-create.txt + + Scenario: Batch written to segment correctly + Tool: Bash + Steps: + 1. Create SegmentWriter in temp dir + 2. Encode a batch with 5 entries + 3. Call AppendBatch(encodedBatch) + 4. Sync + Close + 5. Read file back → verify batch can be recovered + Expected Result: Written batch recoverable from segment file + Evidence: .omo/evidence/task-10-segment-write.txt + ``` + + **Commit**: YES (with Task 11) + - Message: `feat(wal): implement segment writer with rotation` + +- [ ] 11. Segment rotation + durable-ready protocol + + **What to do**: + - 创建 `wal/segment_manager.go`：管理 segment 的轮转和生命周期 + - 写入 batch 前检查 `RemainingPayload()` 是否足够容纳整个 batch + - 不足时： + 1. 当前 segment 完成（在 batch 边界） + 2. 创建新 segment：segment-N+1.wal.tmp → header → fsync → rename → fsync dir + 3. 新 segment 的 startSequence = 下一个 expected sequence + 4. 切换 active segment + - 不允许 batch 跨 segment + - CURRENT 文件 best-effort 更新（temp + rename） + - CURRENT 更新失败不影响已 durable-ready 的 segment + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on Task 10) + - **Parallel Group**: Sub-Wave 3b (solo — after segment writer) + - **Blocks**: 13 + - **Blocked By**: 10 + + **References**: + **Pattern References**: + - `docs/design.md:392-453` — Segment Rotation 约束 + - `docs/design.md:247-278` — WAL 元数据持久化协议 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Segment rotation triggers when payload exceeds remaining space + Tool: Bash + Steps: + 1. Create a test WalConfig with: MaxBatchSize=256, MaxBatchEntries=10, + BlockSize=256, MaxSegmentSize=4096 (satisfies invariant: 4096 ≥ maxEncodedWalBatchSize + since MaxBatchSize=256 → maxEncoded ≈ 256 + 4*10 + overhead < 512) + 2. Create SegmentManager with this config + 3. Write small batches until segment-0 is nearly full (~3800 bytes written) + 4. Write one more batch that would exceed remaining → triggers rotation + 5. Verify segment-1.wal exists with correct header + 6. Verify batch was written to segment-1, not segment-0 + Expected Result: Rotation creates new segment, batch not split across segments + Failure Indicators: segment-1.wal does not exist, or batch data spans both segment files + Evidence: .omo/evidence/task-11-rotation.txt + + Scenario: CURRENT file updated after rotation + Tool: Bash + Steps: + 1. Trigger rotation using same test config as above + 2. Read CURRENT file → verify it points to latest segment filename + Expected Result: CURRENT reflects latest active segment filename + Failure Indicators: CURRENT still points to segment-0, or file content is stale + Evidence: .omo/evidence/task-11-current.txt + ``` + + **Commit**: YES (with Task 10) + +- [x] 12. Sequence manager + + **What to do**: + - 创建 `wal/sequence.go`：`SequenceManager` + - `nextSequence atomic.Uint64` — 下一个待分配 sequence + - `publishedSequence atomic.Uint64` — 已发布的连续 high-water mark + - `durableSequence atomic.Uint64` — 已满足 durable 条件的 high-water mark + - `AllocateBatch(count uint32) (baseSequence uint64, err error)` — checked arithmetic：`baseSequence + count - 1` 不溢出 uint64。溢出 → 返回 ErrSequenceExhausted + - `Publish(seq uint64)` — Store publishedSequence（在所有 entry 节点发布后调用） + - `MarkDurable(endState SegmentEndState)` — 推进 durableSequence + - `Published() uint64` — Load publishedSequence + - `Durable() uint64` — Load durableSequence + - `NextSequence() uint64` — Load nextSequence + - 恢复后初始化：`NewSequenceManager(recoveredSequence uint64)` + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Wave 1b (with Tasks 2, 3, 4, 5, 18 — all depend only on Task 1) + - **Blocks**: 13 + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:486-499` — Sequence 运算和溢出检查 + - `docs/design.md:205-223` — durableSequence 推进逻辑 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Sequence allocation returns consecutive bases + Tool: Bash + Steps: + 1. Create TestSequenceAllocation in wal/sequence_test.go + 2. NewSequenceManager(0) + 3. AllocateBatch(5) → base=0 + 4. AllocateBatch(3) → base=5 + 5. AllocateBatch(1) → base=8 + Expected Result: Consecutive, no gaps + Evidence: .omo/evidence/task-12-seq-alloc.txt + + Scenario: Sequence overflow returns ErrSequenceExhausted + Tool: Bash + Steps: + 1. NewSequenceManager(math.MaxUint64 - 2) + 2. AllocateBatch(5) → err (would overflow) + Expected Result: ErrSequenceExhausted returned + Evidence: .omo/evidence/task-12-seq-overflow.txt + ``` + + **Commit**: YES (with Task 13) + +- [ ] 13. Commit queue + WAL writer main loop + + **What to do**: + - 创建 `wal/commit_queue.go`：`CommitQueue` — 写请求队列 + - 每个 request 包含 entries、result channel + - `Submit(entries []*WalEntry) chan WriteResult` — 非阻塞提交 + - `Collect(timeout time.Duration) []*CommitRequest` — 收集当前队列中的请求 + - 创建 `wal/writer.go`：`WalWriter` — 核心写入循环 + - 单 goroutine 运行 main loop + - Group commit：收集 queue 中的请求 → 组装 batch → 触发条件（500µs 或 32KB） + - 严格遵循 design.md steps ①-⑪： + 1. 从 commit queue 收集写入 + 2. 组装 WAL Batch + 3. ValidateBatchLimits → 失败则普通错误（不分配 sequence） + 4. 计算 MemTable Arena 预留量 → 不足则 freeze/switch 或等待 + 5. AllocateBatch → 分配 baseSequence（此步之后任何失败 → write-stopped） + 6. 私有缓冲编码 WAL Batch + 7. 检查 segment rotation + 8. segmentWriter.AppendBatch(encoded) → 失败 → ErrCommitUnknown + write-stopped + 9. 写入 MemTable pending entries + 10. segmentWriter.Sync() (Always 模式) → 失败 → ErrCommitUnknown + write-stopped + 11. 原子发布 MemTable entries (atomic Pointer store-release) + 12. Publish(baseSequence + entryCount - 1) + 13. 唤醒所有等待的调用方 + - `writeStopped atomic.Bool` — write-stopped 状态 + - `Put(key, value []byte) error` — 提交到 commit queue 并等待结果 + - `Delete(key []byte) error` — 同上 + - `Close() error` — 等待 pending batch 完成 + + **Must NOT do**: + - 不使用 bufio.Writer（直接 os.File.Write） + - 不在 sequence 分配后做可失败的校验 + + **Recommended Agent Profile**: + - **Category**: `deep` + - **Skills**: [`golang-concurrency`] + - `golang-concurrency`: commit queue、group commit、goroutine 生命周期管理 + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on MemTable API from Task 14) + - **Parallel Group**: Sub-Wave 3d (solo — after MemTable complete) + - **Blocks**: 19 + - **Blocked By**: 2, 9, 10, 11, 12, 14 + + **References**: + **Pattern References**: + - `docs/design.md:103-115` — 写入流程 11 步 + - `docs/design.md:119` — 资源前置校验约束 + - `docs/design.md:127-153` — ErrCommitUnknown 和 WAL 副作用边界 + - `docs/design.md:139-153` — per-batch private encode buffer + + **WHY Each Reference Matters**: + - 这是整个 WAL 子系统最复杂的组件，必须严格遵循设计文档的步骤顺序和错误分类 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Single Put writes and syncs correctly + Tool: Bash + Steps: + 1. Create TestWalWriterSinglePut in wal/writer_test.go + 2. Start WalWriter in temp dir + 3. Put("key1", "value1") + 4. Close writer + 5. Read WAL segment → verify batch with 1 entry exists + Expected Result: Entry persisted in WAL + Evidence: .omo/evidence/task-13-writer-single.txt + + Scenario: Multiple concurrent writes group-committed + Tool: Bash + Steps: + 1. Create TestWalWriterGroupCommit in wal/writer_test.go + 2. Start WalWriter + 3. Launch 10 goroutines each doing Put + 4. Wait for all to complete + 5. Close writer + 6. Read WAL → verify entries batched together + Expected Result: Entries grouped in batches, all sequences assigned + Evidence: .omo/evidence/task-13-writer-group.txt + + Scenario: Write-stopped after simulated WAL write failure + Tool: Bash + Steps: + 1. Create a WalWriter with a mock segment writer that fails on 3rd AppendBatch + 2. Write 5 entries sequentially + 3. 3rd write should return ErrCommitUnknown + 4. 4th write should be rejected (write-stopped) + Expected Result: ErrCommitUnknown returned, subsequent writes rejected + Evidence: .omo/evidence/task-13-writer-stopped.txt + ``` + + **Commit**: YES (with Task 12) + - Message: `feat(wal): implement WAL writer with group commit` + +- [ ] 14. MemTable (wrap skiplist + arena + publish/abort) + + **What to do**: + - 创建 `memtable/memtable.go`：`MemTable` 结构体 + - 封装 Arena + SkipList + - `Reserve(entries []ReserveEntry) (totalSize uint32, err error)` — 容量预留（最坏情况：最大层高 next 指针数组 + 对齐 padding）。单个 batch 超过空 MemTable → 返回错误 + - `PutPending(key []byte, value []byte, sequence uint64) error` — 写入 pending entry（Arena 已预留） + - `DeletePending(key []byte, sequence uint64) error` — 写入 pending tombstone + - `Publish(upToSequence uint64)` — 批量发布 sequence <= upToSequence 的 pending entries（atomic store-release） + - `Abort(fromSequence uint64)` — 标记 aborted + - `Get(key []byte, publishedSequence uint64) *GetResult` — 无锁读：先 Load publishedSequence，遍历 skiplist，只返回 sequence <= publishedSequence 且非 aborted 的 entry + - `NewIterator(publishedSequence uint64) *MemTableIterator` — 有序遍历（同样过滤 pending/aborted） + - `ApproximateSize() uint64` + - `UsableCapacity() uint32` — 可用容量（扣除固定元数据） + + **Must NOT do**: + - 不允许 PutPending 在 Reserve 成功后因 Arena 满失败 + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [`golang-concurrency`] + - `golang-concurrency`: 原子发布、无锁读、内存序 + + **Parallelization**: + - **Can Run In Parallel**: NO (solo in sub-wave, Task 13 depends on this) + - **Parallel Group**: Sub-Wave 3c (solo) + - **Blocks**: 13, 19 + - **Blocked By**: 7, 8 + + **References**: + **Pattern References**: + - `docs/design.md:864-879` — Arena 容量预留协议 + - `docs/design.md:881-889` — Flush 过滤规则（pending/aborted 不进入 SSTable） + - `docs/design.md:841-850` — 发布与内存序 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Pending entry invisible to readers + Tool: Bash + Steps: + 1. Create TestMemTablePending in memtable/memtable_test.go + 2. Create MemTable, write PutPending("key1","val1", seq=10) + 3. Get("key1", publishedSequence=5) → Found=false + 4. Publish(upToSequence=10) + 5. Get("key1", publishedSequence=10) → Found=true, Value="val1" + Expected Result: Entry invisible before publish, visible after + Evidence: .omo/evidence/task-14-memtable-pending.txt + + Scenario: Aborted entry invisible to readers + Tool: Bash + Steps: + 1. PutPending("key2","val2", seq=20) + 2. Abort(20) + 3. Publish(upToSequence=30) — publish others but not aborted + 4. Get("key2", publishedSequence=30) → Found=false + Expected Result: Aborted entry never visible + Evidence: .omo/evidence/task-14-memtable-aborted.txt + + Scenario: Reserve accuracy + Tool: Bash + Steps: + 1. Create MemTable with 1024 bytes usable + 2. Reserve 10 entries with 10-byte keys + 10-byte values + 3. Assert reserved size includes node overhead + alignment + 4. Attempt to reserve more than remaining → error + Expected Result: Reserve accounts for full overhead + Evidence: .omo/evidence/task-14-memtable-reserve.txt + ``` + + **Commit**: YES + - Message: `feat(memtable): integrate arena+skiplist with publish/abort` + +- [ ] 15. WAL recovery: scanner + record parser + + **What to do**: + - 创建 `wal/scanner.go`：`SegmentScanner` — 从 WAL 目录扫描 segment 文件 + - `ScanSegments(dir string, recoverySegmentID uint64) ([]*SegmentInfo, error)` — 按 segmentID 排序，过滤 < recoverySegmentID + - `SegmentInfo` 结构体：FilePath, SegmentID, StartSequence（从 header 解析） + - 连续性校验：segmentID 连续、startSequence 衔接 + - 创建 `wal/record_parser.go`：`RecordParser` — Block 级别 Physical Record 解析 + - `ParseBlock(data []byte) ([]*PhysicalRecord, error)` — 解析 block 内所有 records + - Padding 校验：剩余 < 7 bytes 时全 0 + - Short block 处理：最后 segment 的最后 block 可以短于 32KB + - 错误分类辅助：`IsTailCorruption(err error) bool` + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on 4, 10) + - **Parallel Group**: Sub-Wave 4a (solo) + - **Blocks**: 16 + - **Blocked By**: 4, 10 + + **References**: + **Pattern References**: + - `docs/design.md:597-665` — Recovery 扫描流程、segment 连续性校验 + - `docs/design.md:669-691` — Physical Record 解析规则 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Segment scanner finds and orders segments correctly + Tool: Bash + Steps: + 1. Create temp dir with segment-0.wal, segment-2.wal, segment-1.wal + 2. ScanSegments(dir, 0) → returns 3 segments ordered by ID + 3. ScanSegments(dir, 1) → returns segments 1,2 only + Expected Result: Correct ordering and filtering + Evidence: .omo/evidence/task-15-scanner.txt + + Scenario: Record parser handles valid blocks + Tool: Bash + Steps: + 1. Create a 32KB block with 3 Physical Records + 2. ParseBlock → verify 3 records parsed with correct payloads + Expected Result: All records extracted + Evidence: .omo/evidence/task-15-parser.txt + ``` + + **Commit**: YES (with Tasks 16, 17) + +- [ ] 16. WAL recovery: fragment collector + batch replay + + **What to do**: + - 创建 `wal/recovery.go`：恢复核心逻辑 + - 使用 Task 6 的 `FragmentCollector` 进行 fragment 重组 + - `BatchReplayer` 接口：`ReplayPut(key, value []byte, sequence uint64)` / `ReplayDelete(key []byte, sequence uint64)` + - `ReplayBatch(batch *WalBatch, expectedSequence uint64, replayer BatchReplayer) (nextSequence uint64, err error)` + - 校验 Batch Header：flags、entryCount (0 < N <= max)、entriesSize、baseSequence == expectedSequence + - 逐条解析 Entry，校验 opType、valueKind、keyLen、valLen + - 调用 replayer 回调 + - 返回 nextSequence = expectedSequence + entryCount + - Sequence 溢出检查：`baseSequence + entryCount - 1` 不溢出 + + **Recommended Agent Profile**: + - **Category**: `deep` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on 6, 15) + - **Parallel Group**: Sub-Wave 4b (solo — after scanner) + - **Blocks**: 17 + - **Blocked By**: 6, 15 + + **References**: + **Pattern References**: + - `docs/design.md:733-781` — WAL Batch 校验与重放规则 + - `docs/design.md:486-499` — Sequence 溢出检查 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Valid batch replayed correctly + Tool: Bash + Steps: + 1. Create TestBatchReplay in wal/recovery_test.go + 2. Encode a batch: baseSeq=0, 3 entries (Put a, Put b, Delete c) + 3. ReplayBatch with expectedSequence=0 and mock replayer + 4. Assert replayer received 3 calls in order with correct args + Expected Result: All entries replayed with correct sequences + Evidence: .omo/evidence/task-16-batch-replay.txt + + Scenario: Batch with wrong baseSequence rejected + Tool: Bash + Steps: + 1. Encode batch with baseSeq=100 + 2. ReplayBatch with expectedSequence=50 → error + Expected Result: Sequence mismatch error + Evidence: .omo/evidence/task-16-seq-mismatch.txt + ``` + + **Commit**: YES (with Tasks 15, 17) + +- [ ] 17. WAL recovery: tail truncation + main flow + + **What to do**: + - 创建 `wal/truncation.go`：尾部截断持久化 + - `TruncateTail(segmentPath string, truncateOffset int64) error` — ftruncate + fsync + - `DeleteEmptySegments(dir string, fromSegmentID uint64, expectedSequence uint64) error` — 删除空 segment + fsync dir + - 复用 Task 16 定义的 `BatchReplayer` 接口和 `ReplayBatch` 函数（同在 `wal/recovery.go`） + - `Recover(dir string, manifest *manifest.Manifest, cfg *config.WalConfig, replayer BatchReplayer) (*RecoveryResult, error)` + - `BatchReplayer` 由 Task 16 定义（interface{ ReplayPut/ReplayDelete }），Task 19 传入 MemTable 适配实现 + - 主流程严格遵循 design.md lines 622-636： + 1. 读取 MANIFEST → recoverySegmentID + 2. ScanSegments → 过滤 + 3. 校验连续性 + 4. 逐 segment：校验 header → 逐 block 解析 → fragment 重组 → 调用 Task 16 的 `ReplayBatch(batch, expectedSeq, replayer)` 重放 + 5. 处理尾部异常（最后 segment → truncation，中间 segment → 报错） + 6. 持久化截断 + - 返回 `RecoveryResult`：RecoveredSequence, NextSequence, PublishedSequence + - 错误分类： + - 尾部（最后 segment）：header 半写、length 越界、CRC 失败、非 0 padding → 截断 + - 中间（非最后 segment）：同样情况 → 报错 + - CRC-valid batch 内部错误（entryCount 越界等）→ 报错（不可截断） + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: NO (depends on 15, 16) + - **Parallel Group**: Sub-Wave 4c (solo — after batch replay) + - **Blocks**: 19 + - **Blocked By**: 15, 16 + + **References**: + **Pattern References**: + - `docs/design.md:622-636` — Recovery 扫描流程 + - `docs/design.md:640-665` — Segment 连续性校验 + - `docs/design.md:783-800` — 尾部截断持久化 + - `docs/design.md:694-703` — 错误分类（尾部 vs 中间） + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Normal recovery from clean WAL + Tool: Bash + Steps: + 1. Create temp dir, write 2 segments with valid batches + 2. Recover(dir, manifest) → success + 3. Assert RecoveredSequence == last batch end sequence + 4. Assert all entries replayed via mock replayer + Expected Result: Full recovery, no data loss + Evidence: .omo/evidence/task-17-recovery-normal.txt + + Scenario: Recovery with tail corruption truncates + Tool: Bash + Steps: + 1. Write valid segment + partial write at end (corrupt last record) + 2. Recover → success (truncated to last complete batch) + 3. Verify segment file was truncated + Expected Result: Tail corruption handled, complete batches preserved + Evidence: .omo/evidence/task-17-recovery-tail.txt + + Scenario: Recovery with middle corruption reports error + Tool: Bash + Steps: + 1. Write 3 segments, corrupt the middle one (bad CRC) + 2. Recover → error containing "corrupt" or "damaged" + Expected Result: Middle corruption detected, recovery fails + Evidence: .omo/evidence/task-17-recovery-middle.txt + ``` + + **Commit**: YES (with Tasks 15, 16) + +- [x] 18. MANIFEST stub + CURRENT file + + **What to do**: + - 创建 `manifest/manifest.go`：`Manifest` 最小 stub + - 仅存储 `recoverySegmentID uint64` + - `Load(dir string) (*Manifest, error)` — 读取 MANIFEST 文件 + - `Save(dir string, recoverySegmentID uint64) error` — temp + rename 原子写入 + - 首次创建：如果 MANIFEST 不存在，创建初始文件（recoverySegmentID=0） + - 创建 `manifest/current.go`：CURRENT 文件 + - `WriteCurrent(dir string, segmentID uint64) error` — temp + rename，best-effort + - `ReadCurrent(dir string) (segmentID uint64, ok bool)` — 读取，不存在或损坏返回 ok=false + - 格式：简单的文本文件或二进制（Phase 1 选择简单文本） + + **Recommended Agent Profile**: + - **Category**: `quick` + - **Skills**: [] + + **Parallelization**: + - **Can Run In Parallel**: YES + - **Parallel Group**: Wave 1b (with Tasks 2, 3, 4, 5, 12 — all depend only on Task 1) + - **Blocks**: 19 + - **Blocked By**: 1 + + **References**: + **Pattern References**: + - `docs/design.md:603-619` — CURRENT / MANIFEST 权威性 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: MANIFEST save and load round-trip + Tool: Bash + Steps: + 1. Save(dir, 42) + 2. Load(dir) → assert recoverySegmentID == 42 + Expected Result: Round-trip correct + Evidence: .omo/evidence/task-18-manifest.txt + + Scenario: CURRENT best-effort handles missing file + Tool: Bash + Steps: + 1. ReadCurrent on empty dir → ok=false, no error + Expected Result: Missing CURRENT handled gracefully + Evidence: .omo/evidence/task-18-current-missing.txt + ``` + + **Commit**: YES + - Message: `feat: implement MANIFEST stub and CURRENT file` + +- [ ] 19. DB integration: Open/Close/Put/Delete/Get + + **What to do**: + - 创建 `db.go`：`DB` 结构体和核心 API + - `DB` 字段：walWriter, memTable, immutableTables, seqMgr, manifest, config, dir, writeStopped + - `Open(dir string, opts ...Option) (*DB, error)`: + 1. Load or create MANIFEST + 2. Recover WAL → 重建 MemTable（恢复的 entries 标记为 published） + 3. 设置 nextSequence、publishedSequence + 4. 启动 WAL writer goroutine + - `Close() error`: + 1. 停止 WAL writer（等待 pending batch 完成） + 2. Sync WAL + 3. Save MANIFEST（recoverySegmentID stays 0 in Phase 1 — no SSTable flush means all segments must be replayed） + 4. Close segment writer + - `Put(key, value []byte) error`: + 1. ValidateBatchLimits (key/value size) + 2. Submit to WAL writer → wait for result + 3. 返回 nil 或 ErrCommitUnknown 或 write-stopped error + - `Delete(key []byte) error` — 同上，opType=Delete + - `Get(key []byte) (GetResult, error)`: + 1. Load publishedSequence + 2. 查 active MemTable → Immutable #1 → Immutable #2 + 3. 只返回 sequence <= publishedSequence 且非 aborted + 4. Tombstone → Found=false + 5. 未找到 → Found=false + - `GetResult` 结构体：Value []byte, Found bool（参照 design.md §3.4） + - `GetDurableSequence() uint64` + - `IsWriteStopped() bool` + - MemTable freeze + switch： + - Put 前检查容量，不足时 freeze + - Immutable 队列满时阻塞 + - Phase 1：Immutable 不 flush 到 SSTable，仅持有在内存 + - Options pattern：`type Option func(*config.WalConfig)` + + **Recommended Agent Profile**: + - **Category**: `deep` + - **Skills**: [`golang-design-patterns`] + - `golang-design-patterns`: Options pattern for configuration + + **Parallelization**: + - **Can Run In Parallel**: NO (integrates all deps) + - **Parallel Group**: Sub-Wave 4d (solo — after recovery + WAL writer + MemTable + MANIFEST) + - **Blocks**: 20, 21 + - **Blocked By**: 13, 14, 17, 18 + + **References**: + **Pattern References**: + - `docs/design.md:901-921` — Get API 和 GetResult 定义 + - `docs/design.md:928-941` — 读路径优先级 + - `docs/design.md:852-862` — MemTable 生命周期 + + **WHY Each Reference Matters**: + - 这是所有模块的集成点，必须正确协调 WAL writer、MemTable、recovery 和 MANIFEST + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Open creates fresh DB, Put+Get round-trip + Tool: Bash + Steps: + 1. Create temp dir + 2. Open(dir) → success + 3. Put("hello", "world") → nil + 4. Get("hello") → {Found:true, Value:"world"} + 5. Close() → nil + Expected Result: Full round-trip works + Evidence: .omo/evidence/task-19-open-put-get.txt + + Scenario: Close and reopen preserves data + Tool: Bash + Steps: + 1. Open(dir), Put("k1","v1"), Put("k2","v2"), Close() + 2. Open(dir) → recovery runs + 3. Get("k1") → {Found:true, Value:"v1"} + 4. Get("k2") → {Found:true, Value:"v2"} + 5. Close() + Expected Result: Data survives close/reopen + Evidence: .omo/evidence/task-19-reopen.txt + + Scenario: Delete makes key not found + Tool: Bash + Steps: + 1. Open, Put("k","v"), Delete("k") + 2. Get("k") → {Found:false} + Expected Result: Delete works correctly + Evidence: .omo/evidence/task-19-delete.txt + + Scenario: Empty value distinguished from not found + Tool: Bash + Steps: + 1. Put("empty", []byte{}) + 2. Get("empty") → {Found:true, Value:[]byte{}} + 3. Get("nonexist") → {Found:false} + Expected Result: Found distinguishes empty value from missing key + Evidence: .omo/evidence/task-19-empty-value.txt + + Scenario: Concurrent Put and Get work correctly + Tool: Bash + Steps: + 1. Open + 2. Launch 10 writers (Put "key-{i}", "val-{i}") + 3. Launch 10 readers (Get "key-{i}") + 4. go test -race + Expected Result: No race conditions + Evidence: .omo/evidence/task-19-concurrent.txt + ``` + + **Commit**: YES + - Message: `feat: integrate DB with Open/Close/Put/Delete/Get` + +- [ ] 20. End-to-end + crash recovery tests + + **What to do**: + - 创建 `db_test.go`：集成测试 + - `TestWALRoundtrip` — 写 N 条 → Close → Open → 验证全部 + - `TestRecoveryPartialWrite` — 模拟崩溃：写入数据 → 在 WAL 写入中途截断文件 → Recover → 验证完整 batch 保留 + - `TestRecoveryMultiSegment` — 多 segment 恢复 + - `TestRecoveryTailCorruption` — 尾部损坏修复 + - `TestWriteStoppedAfterFailure` — 模拟写入失败 → ErrCommitUnknown → write-stopped + - `TestGetSemantics` — Found vs empty value vs not found + - `TestSequenceExhaustion` — 虽然不可达，但验证 sequence 溢出返回正确错误 + - `TestConcurrentReadWrite` — 多 goroutine 并发读写 + - `TestLargeBatchSegmentRotation` — 大量写入触发多次 segment 轮转 + - `TestMemTableFreezeNoSSTable` — 验证 MemTable 满时冻结，不刷盘 + + **Recommended Agent Profile**: + - **Category**: `deep` + - **Skills**: [`golang-testing`] + - `golang-testing`: Go testing patterns + + **Parallelization**: + - **Can Run In Parallel**: YES (with Task 21) + - **Parallel Group**: Wave 5 (with Task 21) + - **Blocks**: F1-F4 + - **Blocked By**: 19 + + **References**: + **Pattern References**: + - `docs/design.md:694-703` — 错误分类 + - `docs/design.md:783-800` — 尾部截断 + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Full WAL roundtrip with crash simulation + Tool: Bash + Steps: + 1. Open DB, write 1000 keys + 2. Close DB (simulate clean shutdown) + 3. Open DB → verify all 1000 keys readable + Expected Result: All 1000 keys recovered + Evidence: .omo/evidence/task-20-roundtrip.txt + + Scenario: Crash recovery with tail truncation + Tool: Bash + Steps: + 1. Open DB, write 500 keys, Close + 2. Open DB, write 200 more keys + 3. Manually truncate last WAL segment by 100 bytes (simulate crash) + 4. Open DB → verify 500 original keys present, some of 200 may be lost + Expected Result: Complete batches preserved, tail truncated + Evidence: .omo/evidence/task-20-crash.txt + + Scenario: go test -race passes all integration tests + Tool: Bash + Steps: + 1. go test ./... -race -count=1 + Expected Result: Zero race conditions + Evidence: .omo/evidence/task-20-race.txt + ``` + + **Commit**: YES + - Message: `test: add end-to-end and crash recovery tests` + +- [ ] 21. Benchmark suite + + **What to do**: + - 创建 `bench_test.go`：性能基准 + - `BenchmarkSinglePut` — 单线程 Put 吞吐 + - `BenchmarkConcurrentPut` — 多线程 Put 吞吐（2/4/8 goroutines） + - `BenchmarkGet` — Get 延迟 + - `BenchmarkConcurrentGet` — 并发 Get + - `BenchmarkMixedReadWrite` — 混合读写 + - `BenchmarkWALRecovery` — 不同数据量的恢复时间 + - 使用 `b.ReportAllocs()` 跟踪内存分配 + + **Recommended Agent Profile**: + - **Category**: `unspecified-high` + - **Skills**: [`golang-benchmark`] + - `golang-benchmark`: Go benchmarking best practices + + **Parallelization**: + - **Can Run In Parallel**: YES (with Task 20) + - **Parallel Group**: Wave 5 (with Task 20) + - **Blocks**: F1-F4 + - **Blocked By**: 19 + + **References**: + - 无特定设计文档引用（benchmark 是验证手段，不是设计要求） + + **Acceptance Criteria**: + + **QA Scenarios (MANDATORY):** + + ``` + Scenario: Benchmark runs without errors + Tool: Bash + Steps: + 1. go test -bench=. -benchmem ./... + Expected Result: All benchmarks complete, results printed + Evidence: .omo/evidence/task-21-bench.txt + ``` + + **Commit**: YES + - Message: `bench: add WAL performance benchmarks` + +--- + +## Final Verification Wave + +- [ ] F1. **Plan Compliance Audit** — `oracle` + Read the plan end-to-end. For each "Must Have": verify implementation exists (grep for atomic.Uint64, atomic.Pointer, no bufio.Writer on WAL path, GetResult with Found field, etc). For each "Must NOT Have": search codebase for forbidden patterns (SSTable, Compaction, ValueLogPointer write, Periodic sync, bufio.Writer in wal/). Check evidence files exist in .omo/evidence/. Compare deliverables against plan. + **Acceptance Criteria**: + - [ ] All "Must Have" items verified present in codebase + - [ ] All "Must NOT Have" items verified absent + - [ ] Evidence files exist for tasks 1-21 + - [ ] Output: `Must Have [N/N] | Must NOT Have [N/N] | Tasks [21/21] | VERDICT: APPROVE/REJECT` + +- [ ] F2. **Code Quality Review** — `unspecified-high` + Run `go vet ./...` + `golangci-lint run` + `go test ./... -race`. Review all .go files for: `any` type casts, empty catches, fmt.Println in prod, commented-out code, unused imports. Check AI slop: excessive comments, over-abstraction, generic names. + **Acceptance Criteria**: + - [ ] `go vet ./...` passes with zero warnings + - [ ] `golangci-lint run` passes with zero warnings + - [ ] `go test ./... -race` passes with zero races + - [ ] No `fmt.Println` or `log.Println` in non-test files + - [ ] Output: `Build [PASS/FAIL] | Lint [PASS/FAIL] | Tests [N pass/0 fail] | Files [N clean/0 issues] | VERDICT` + +- [ ] F3. **Real Manual QA** — `unspecified-high` + Build the project. Run ALL QA scenarios from tasks 1-21. Test cross-module integration: write data with Put → Close → Open → Get verifies data. Test edge cases: empty value, large batch (9999 entries), concurrent writes from 10 goroutines, segment rotation after filling segment. Save output to `.omo/evidence/final-qa/`. + **Acceptance Criteria**: + - [ ] Every QA scenario from tasks 1-21 executed and passing + - [ ] Cross-module integration: Put → Close → Open → Get round-trip + - [ ] Edge case: empty value returns Found=true + - [ ] Edge case: key not found returns Found=false + - [ ] Edge case: concurrent 10-goroutine write/read with -race clean + - [ ] Output: `Scenarios [N/N pass] | Integration [3/3] | Edge Cases [4 tested] | VERDICT` + +- [ ] F4. **Scope Fidelity Check** — `deep` + For each task: read "What to do", read actual diff (git diff). Verify 1:1 — everything in spec was built (no missing), nothing beyond spec was built (no creep). Check "Must NOT do" compliance per task. Detect cross-task contamination: Task N touching Task M's files without justification. Flag unaccounted files. + **Acceptance Criteria**: + - [ ] Each task's "What to do" matches its actual implementation + - [ ] No SSTable/Compaction/ValueLog/BloomFilter code found + - [ ] No bufio.Writer import in wal/ package + - [ ] No Periodic/Never sync code (only Always) + - [ ] No cross-task file contamination + - [ ] Output: `Tasks [21/21 compliant] | Contamination [CLEAN] | Unaccounted [CLEAN] | VERDICT` + +--- + +## Commit Strategy + +| Tasks | Commit Message | Pre-commit Check | +|-------|---------------|-----------------| +| 1-2 | `feat: initialize project structure and error types` | `go build ./...` | +| 3-5 | `feat(wal): implement WAL entry/record/header codec` | `go test ./wal/... -run TestCodec` | +| 6, 9 | `feat(wal): implement WAL batch codec and resource validation` | `go test ./wal/... -run TestBatch` | +| 7-8 | `feat(memtable): implement Arena allocator and SkipList` | `go test ./memtable/... -race` | +| 10-11 | `feat(wal): implement segment writer with rotation` | `go test ./wal/... -run TestSegment` | +| 12-13 | `feat(wal): implement WAL writer with group commit` | `go test ./wal/... -run TestWriter` | +| 14 | `feat(memtable): integrate arena+skiplist with publish/abort` | `go test ./memtable/... -race` | +| 15-17 | `feat(wal): implement WAL recovery` | `go test ./wal/... -run TestRecovery` | +| 18 | `feat: implement MANIFEST stub and CURRENT file` | `go test ./manifest/...` | +| 19 | `feat: integrate DB with Open/Close/Put/Delete/Get` | `go test ./... -race` | +| 20 | `test: add end-to-end and crash recovery tests` | `go test ./... -race -count=1` | +| 21 | `bench: add WAL performance benchmarks` | `go test -bench=. ./...` | + +--- + +## Success Criteria + +### Verification Commands +```bash +go build ./... # Expected: success, no errors +go vet ./... # Expected: no issues +go test ./... -race -count=1 # Expected: all pass, no races +go test ./... -run TestWALRoundtrip # Expected: write→close→open→read roundtrip +go test ./... -run TestRecovery # Expected: crash recovery correct +go test ./... -run TestGetSemantics # Expected: Found/empty-value distinction +``` + +### Final Checklist +- [ ] All "Must Have" present +- [ ] All "Must NOT Have" absent +- [ ] All tests pass with `-race` +- [ ] WAL roundtrip test passes +- [ ] Crash recovery test passes +- [ ] No bufio.Writer on WAL append path +- [ ] All atomics are typed (atomic.Uint64, atomic.Pointer) +- [ ] Config invariant checked at Open diff --git a/.omo/run-continuation/ses_1465a5321ffevd4inJJZXxFenJ.json b/.omo/run-continuation/ses_1465a5321ffevd4inJJZXxFenJ.json new file mode 100644 index 0000000..b5b1a12 --- /dev/null +++ b/.omo/run-continuation/ses_1465a5321ffevd4inJJZXxFenJ.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_1465a5321ffevd4inJJZXxFenJ", + "updatedAt": "2026-06-12T02:26:27.295Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T02:26:27.295Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_1468f0cc5ffew53IA0I4W6HijM.json b/.omo/run-continuation/ses_1468f0cc5ffew53IA0I4W6HijM.json new file mode 100644 index 0000000..f55509a --- /dev/null +++ b/.omo/run-continuation/ses_1468f0cc5ffew53IA0I4W6HijM.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_1468f0cc5ffew53IA0I4W6HijM", + "updatedAt": "2026-06-12T01:41:18.251Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T01:41:18.251Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_1469860f1ffeAxT7jbUD3T1ZK5.json b/.omo/run-continuation/ses_1469860f1ffeAxT7jbUD3T1ZK5.json new file mode 100644 index 0000000..7497b13 --- /dev/null +++ b/.omo/run-continuation/ses_1469860f1ffeAxT7jbUD3T1ZK5.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_1469860f1ffeAxT7jbUD3T1ZK5", + "updatedAt": "2026-06-12T01:18:59.508Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T01:18:59.508Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_146adbaa4ffehxje0lmKRg4ZpT.json b/.omo/run-continuation/ses_146adbaa4ffehxje0lmKRg4ZpT.json new file mode 100644 index 0000000..07dcbed --- /dev/null +++ b/.omo/run-continuation/ses_146adbaa4ffehxje0lmKRg4ZpT.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_146adbaa4ffehxje0lmKRg4ZpT", + "updatedAt": "2026-06-12T00:56:53.921Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T00:56:53.921Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_146b864adffetMjZkc08p16onz.json b/.omo/run-continuation/ses_146b864adffetMjZkc08p16onz.json new file mode 100644 index 0000000..80f03b5 --- /dev/null +++ b/.omo/run-continuation/ses_146b864adffetMjZkc08p16onz.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_146b864adffetMjZkc08p16onz", + "updatedAt": "2026-06-12T00:44:22.791Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T00:44:22.791Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_146c330caffehDbiMQ7M0jL3Il.json b/.omo/run-continuation/ses_146c330caffehDbiMQ7M0jL3Il.json new file mode 100644 index 0000000..1766f92 --- /dev/null +++ b/.omo/run-continuation/ses_146c330caffehDbiMQ7M0jL3Il.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_146c330caffehDbiMQ7M0jL3Il", + "updatedAt": "2026-06-12T00:31:46.022Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-12T00:31:46.022Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_149585098ffeVZ53676PRrr2ic.json b/.omo/run-continuation/ses_149585098ffeVZ53676PRrr2ic.json new file mode 100644 index 0000000..74eeaef --- /dev/null +++ b/.omo/run-continuation/ses_149585098ffeVZ53676PRrr2ic.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_149585098ffeVZ53676PRrr2ic", + "updatedAt": "2026-06-11T12:31:44.016Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T12:31:44.016Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_14960b98effe7YjeN56yvxsoN3.json b/.omo/run-continuation/ses_14960b98effe7YjeN56yvxsoN3.json new file mode 100644 index 0000000..1d30e4d --- /dev/null +++ b/.omo/run-continuation/ses_14960b98effe7YjeN56yvxsoN3.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_14960b98effe7YjeN56yvxsoN3", + "updatedAt": "2026-06-11T12:21:18.579Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T12:21:18.579Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_14967e191ffeLfQYyfTQHP5TlH.json b/.omo/run-continuation/ses_14967e191ffeLfQYyfTQHP5TlH.json new file mode 100644 index 0000000..c34e22d --- /dev/null +++ b/.omo/run-continuation/ses_14967e191ffeLfQYyfTQHP5TlH.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_14967e191ffeLfQYyfTQHP5TlH", + "updatedAt": "2026-06-11T12:12:33.047Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T12:12:33.047Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_149a13a61ffehpTGChldqMe0do.json b/.omo/run-continuation/ses_149a13a61ffehpTGChldqMe0do.json new file mode 100644 index 0000000..a9a2e54 --- /dev/null +++ b/.omo/run-continuation/ses_149a13a61ffehpTGChldqMe0do.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_149a13a61ffehpTGChldqMe0do", + "updatedAt": "2026-06-11T11:11:39.805Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T11:11:39.805Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_149e4f0c5ffe5hf5hErOGwmaO7.json b/.omo/run-continuation/ses_149e4f0c5ffe5hf5hErOGwmaO7.json new file mode 100644 index 0000000..58301d1 --- /dev/null +++ b/.omo/run-continuation/ses_149e4f0c5ffe5hf5hErOGwmaO7.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_149e4f0c5ffe5hf5hErOGwmaO7", + "updatedAt": "2026-06-11T10:01:28.529Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T10:01:28.529Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_149f503d8ffeCitCeNNhuI5uPD.json b/.omo/run-continuation/ses_149f503d8ffeCitCeNNhuI5uPD.json new file mode 100644 index 0000000..8e7c633 --- /dev/null +++ b/.omo/run-continuation/ses_149f503d8ffeCitCeNNhuI5uPD.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_149f503d8ffeCitCeNNhuI5uPD", + "updatedAt": "2026-06-11T09:38:47.486Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-11T09:38:47.486Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_1551b5cb8ffeRq3O3tTE6gADNa.json b/.omo/run-continuation/ses_1551b5cb8ffeRq3O3tTE6gADNa.json new file mode 100644 index 0000000..27a9ac4 --- /dev/null +++ b/.omo/run-continuation/ses_1551b5cb8ffeRq3O3tTE6gADNa.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_1551b5cb8ffeRq3O3tTE6gADNa", + "updatedAt": "2026-06-09T05:49:57.317Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-09T05:49:57.317Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_15535520fffenpCk4v55tlxfPN.json b/.omo/run-continuation/ses_15535520fffenpCk4v55tlxfPN.json new file mode 100644 index 0000000..5ec4273 --- /dev/null +++ b/.omo/run-continuation/ses_15535520fffenpCk4v55tlxfPN.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_15535520fffenpCk4v55tlxfPN", + "updatedAt": "2026-06-09T05:14:00.942Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-09T05:14:00.942Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_155c694ebffeUxfvz2X0kcpMQz.json b/.omo/run-continuation/ses_155c694ebffeUxfvz2X0kcpMQz.json new file mode 100644 index 0000000..4771a5b --- /dev/null +++ b/.omo/run-continuation/ses_155c694ebffeUxfvz2X0kcpMQz.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_155c694ebffeUxfvz2X0kcpMQz", + "updatedAt": "2026-06-09T03:20:21.146Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-09T03:20:21.146Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_162be27e5ffePD7UoGttrWFm7T.json b/.omo/run-continuation/ses_162be27e5ffePD7UoGttrWFm7T.json new file mode 100644 index 0000000..fdd2ebf --- /dev/null +++ b/.omo/run-continuation/ses_162be27e5ffePD7UoGttrWFm7T.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_162be27e5ffePD7UoGttrWFm7T", + "updatedAt": "2026-06-06T14:13:39.321Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-06T14:13:39.321Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_16f5a4050ffe4VOJm8EOoPcnZV.json b/.omo/run-continuation/ses_16f5a4050ffe4VOJm8EOoPcnZV.json new file mode 100644 index 0000000..6c68e14 --- /dev/null +++ b/.omo/run-continuation/ses_16f5a4050ffe4VOJm8EOoPcnZV.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_16f5a4050ffe4VOJm8EOoPcnZV", + "updatedAt": "2026-06-04T03:24:44.020Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-04T03:24:44.020Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_16f63a221ffew3fp5TM11NaRDD.json b/.omo/run-continuation/ses_16f63a221ffew3fp5TM11NaRDD.json new file mode 100644 index 0000000..63ffe70 --- /dev/null +++ b/.omo/run-continuation/ses_16f63a221ffew3fp5TM11NaRDD.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_16f63a221ffew3fp5TM11NaRDD", + "updatedAt": "2026-06-04T03:18:04.363Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-04T03:18:04.363Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_17337fda2fferaZJ6tpXI2Cw44.json b/.omo/run-continuation/ses_17337fda2fferaZJ6tpXI2Cw44.json new file mode 100644 index 0000000..1d784a5 --- /dev/null +++ b/.omo/run-continuation/ses_17337fda2fferaZJ6tpXI2Cw44.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_17337fda2fferaZJ6tpXI2Cw44", + "updatedAt": "2026-06-04T03:25:22.600Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-04T03:25:22.600Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_1738cec10ffex8teaUh336WrCD.json b/.omo/run-continuation/ses_1738cec10ffex8teaUh336WrCD.json new file mode 100644 index 0000000..5ce4dd1 --- /dev/null +++ b/.omo/run-continuation/ses_1738cec10ffex8teaUh336WrCD.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_1738cec10ffex8teaUh336WrCD", + "updatedAt": "2026-06-03T08:41:49.094Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-06-03T08:41:49.094Z" + } + } +} \ No newline at end of file diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..b464c05 --- /dev/null +++ b/config/config.go @@ -0,0 +1,205 @@ +// Package config defines configuration types and validation for the go-kv storage engine. +package config + +import ( + "fmt" + "math" +) + +// WAL format constants derived from the binary layout specification. +const ( + walFileHeaderSize uint64 = 32 + walBlockSize uint32 = 32 * 1024 // 32KB + physicalRecordHeaderSize uint64 = 7 + walBatchHeaderSize uint64 = 18 // flags(2) + baseSequence(8) + entryCount(4) + entriesSize(4) +) + +// WalConfig holds configuration for the Write-Ahead Log subsystem. +// Zero-value WalConfig is valid and uses defaults; call Validate() to apply +// defaults and verify invariants. +type WalConfig struct { + // MaxSegmentSize is the maximum size of a single WAL segment file in bytes. + // Default: 64MB. Must be large enough to hold the largest possible WAL Batch. + MaxSegmentSize uint64 + + // BlockSize is the WAL block size in bytes. + // Default: 32KB. + BlockSize uint32 + + // SyncMode controls when WAL writes are flushed to disk. + // Phase 1 only supports "always". + SyncMode string + + // MaxBatchEntries is the maximum number of entries in a single WAL Batch. + // Default: 10000. + MaxBatchEntries uint32 + + // MaxBatchSize is the maximum total size of WAL Batch entries in bytes. + // Default: 4MB. + MaxBatchSize uint32 + + // MaxKeyBytes is the maximum size of a single key in bytes. + // Default: 4KB. + MaxKeyBytes uint32 + + // MaxInlineValue is the maximum size of an inline value in bytes. + // Values larger than this must use ValueLogPointer. + // Default: 4KB. + MaxInlineValue uint32 + + // MemTableSize is the target MemTable size in bytes before triggering flush. + // Default: 64MB. + MemTableSize uint32 + + // MaxImmutableCount is the maximum number of immutable MemTables allowed + // before writes are stalled. Default: 3. + MaxImmutableCount int +} + +// Defaults returns a WalConfig populated with production defaults. +func Defaults() WalConfig { + return WalConfig{ + MaxSegmentSize: 64 * 1024 * 1024, // 64MB + BlockSize: 32 * 1024, // 32KB + SyncMode: "always", + MaxBatchEntries: 10000, + MaxBatchSize: 4 * 1024 * 1024, // 4MB + MaxKeyBytes: 4 * 1024, // 4KB + MaxInlineValue: 4 * 1024, // 4KB + MemTableSize: 64 * 1024 * 1024, // 64MB + MaxImmutableCount: 3, + } +} + +// applyDefaults fills zero-valued fields with production defaults. +func (c *WalConfig) applyDefaults() { + d := Defaults() + if c.MaxSegmentSize == 0 { + c.MaxSegmentSize = d.MaxSegmentSize + } + if c.BlockSize == 0 { + c.BlockSize = d.BlockSize + } + if c.SyncMode == "" { + c.SyncMode = d.SyncMode + } + if c.MaxBatchEntries == 0 { + c.MaxBatchEntries = d.MaxBatchEntries + } + if c.MaxBatchSize == 0 { + c.MaxBatchSize = d.MaxBatchSize + } + if c.MaxKeyBytes == 0 { + c.MaxKeyBytes = d.MaxKeyBytes + } + if c.MaxInlineValue == 0 { + c.MaxInlineValue = d.MaxInlineValue + } + if c.MemTableSize == 0 { + c.MemTableSize = d.MemTableSize + } + if c.MaxImmutableCount == 0 { + c.MaxImmutableCount = d.MaxImmutableCount + } +} + +// Validate applies defaults and verifies that all configuration invariants hold. +// The key invariant ensures that the largest possible WAL Batch can fit into +// an empty WAL segment: +// +// maxWalSegmentPayload >= maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding +// +// All arithmetic is checked for overflow. +func (c *WalConfig) Validate() error { + c.applyDefaults() + + if c.SyncMode != "always" { + return fmt.Errorf("config: SyncMode %q not supported (Phase 1: only \"always\")", c.SyncMode) + } + + if c.MaxImmutableCount < 1 { + return fmt.Errorf("config: MaxImmutableCount must be >= 1, got %d", c.MaxImmutableCount) + } + + // --- Checked arithmetic invariant validation --- + // Mirrors the derivation in docs/design.md § WAL Segment Rotation. + + blockSize := uint64(c.BlockSize) + prHeaderSize := physicalRecordHeaderSize + batchHeaderSize := walBatchHeaderSize + maxBatchEntriesSize := uint64(c.MaxBatchSize) + + // maxEncodedWalBatchSize = batchHeaderSize + maxBatchEntriesSize + maxEncodedWalBatchSize, err := safeAdd(batchHeaderSize, maxBatchEntriesSize) + if err != nil { + return fmt.Errorf("config: WAL batch size overflow: %w", err) + } + + // maxPhysicalRecordPayload = blockSize - prHeaderSize + if blockSize <= prHeaderSize { + return fmt.Errorf("config: BlockSize %d must be > physical record header size %d", blockSize, prHeaderSize) + } + maxPhysicalRecordPayload := blockSize - prHeaderSize + + // maxPhysicalRecordCount = ceil(maxEncodedWalBatchSize / maxPhysicalRecordPayload) + maxPhysicalRecordCount := divCeil(maxEncodedWalBatchSize, maxPhysicalRecordPayload) + + // worstCasePhysicalRecordOverhead = maxPhysicalRecordCount * prHeaderSize + worstCasePhysicalRecordOverhead, err := safeMul(maxPhysicalRecordCount, prHeaderSize) + if err != nil { + return fmt.Errorf("config: physical record overhead overflow: %w", err) + } + + // worstCaseBlockPadding = blockSize - 1 (at most one partial block of padding) + // From design doc: worstCaseBlockPadding = 7 bytes with default block size. + // Generalized: blockSize - maxPhysicalRecordPayload = prHeaderSize + worstCaseBlockPadding := prHeaderSize + + // minWalSegmentPayload = maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding + partial, err := safeAdd(maxEncodedWalBatchSize, worstCasePhysicalRecordOverhead) + if err != nil { + return fmt.Errorf("config: segment payload calculation overflow: %w", err) + } + minWalSegmentPayload, err := safeAdd(partial, worstCaseBlockPadding) + if err != nil { + return fmt.Errorf("config: segment payload calculation overflow: %w", err) + } + + // maxWalSegmentPayload = MaxSegmentSize - walFileHeaderSize + if c.MaxSegmentSize <= walFileHeaderSize { + return fmt.Errorf("config: MaxSegmentSize %d must be > WAL file header size %d", + c.MaxSegmentSize, walFileHeaderSize) + } + maxWalSegmentPayload := c.MaxSegmentSize - walFileHeaderSize + + if maxWalSegmentPayload < minWalSegmentPayload { + return fmt.Errorf("config: MaxSegmentSize %d too small: "+ + "segment payload (%d) < minimum required (%d); "+ + "need MaxSegmentSize >= %d", + c.MaxSegmentSize, maxWalSegmentPayload, minWalSegmentPayload, + minWalSegmentPayload+walFileHeaderSize) + } + + return nil +} + +// safeAdd returns a + b or an error if the result overflows uint64. +func safeAdd(a, b uint64) (uint64, error) { + if a > math.MaxUint64-b { + return 0, fmt.Errorf("uint64 overflow: %d + %d", a, b) + } + return a + b, nil +} + +// safeMul returns a * b or an error if the result overflows uint64. +func safeMul(a, b uint64) (uint64, error) { + if a != 0 && b > math.MaxUint64/a { + return 0, fmt.Errorf("uint64 overflow: %d * %d", a, b) + } + return a * b, nil +} + +// divCeil returns ceil(a / b) for b > 0. +func divCeil(a, b uint64) uint64 { + return (a + b - 1) / b +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..d1a3dfe --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,32 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateDefaults(t *testing.T) { + cfg := WalConfig{} + err := cfg.Validate() + require.NoError(t, err, "default WalConfig should pass validation") + + d := Defaults() + assert.Equal(t, d.MaxSegmentSize, cfg.MaxSegmentSize, "MaxSegmentSize should be defaulted") + assert.Equal(t, d.BlockSize, cfg.BlockSize, "BlockSize should be defaulted") + assert.Equal(t, d.SyncMode, cfg.SyncMode, "SyncMode should be defaulted") + assert.Equal(t, d.MaxBatchEntries, cfg.MaxBatchEntries, "MaxBatchEntries should be defaulted") + assert.Equal(t, d.MaxBatchSize, cfg.MaxBatchSize, "MaxBatchSize should be defaulted") + assert.Equal(t, d.MaxKeyBytes, cfg.MaxKeyBytes, "MaxKeyBytes should be defaulted") + assert.Equal(t, d.MaxInlineValue, cfg.MaxInlineValue, "MaxInlineValue should be defaulted") + assert.Equal(t, d.MemTableSize, cfg.MemTableSize, "MemTableSize should be defaulted") + assert.Equal(t, d.MaxImmutableCount, cfg.MaxImmutableCount, "MaxImmutableCount should be defaulted") +} + +func TestValidateSegmentTooSmall(t *testing.T) { + cfg := WalConfig{MaxSegmentSize: 1024} + err := cfg.Validate() + require.Error(t, err, "MaxSegmentSize=1024 should fail validation") + assert.Contains(t, err.Error(), "too small") +} diff --git a/docs/.markdownlint.json b/docs/.markdownlint.json new file mode 100644 index 0000000..74f1189 --- /dev/null +++ b/docs/.markdownlint.json @@ -0,0 +1,7 @@ +{ + "MD013": { + "line_length": 120, + "code_blocks": false, + "tables": false + } +} diff --git a/docs/issues/oracle-wal-3.2-review.md b/docs/issues/oracle-wal-3.2-review.md new file mode 100644 index 0000000..c089715 --- /dev/null +++ b/docs/issues/oracle-wal-3.2-review.md @@ -0,0 +1,297 @@ +# WAL Section 3.2 Oracle 审核报告 — Issues 清单 + +> 来源：Oracle 对 `docs/design.md` Section 3.2 WAL 的架构审核 +> 日期：2026-06-09 + +--- + +## Critical Issues + +### C1. WAL write failure 语义过于简化：`write()` 失败后 bytes 可能已落盘 + +**严重程度**: Critical +**位置**: Section 3.2 "写入流程" 步骤 ⑤ 及后续错误处理段落（~line 117） + +**问题描述**: + +当前设计将 WAL encode/write 失败统一当作 "definitely failed" 返回普通错误。但实际上存在两种不同情况： + +1. **Encode 失败**（未触及 syscall）：确实是 definitely failed，可以安全返回普通错误 +2. **`write()` 失败**（bytes 可能已进入 OS page cache 或部分写入文件）：不是 definitely failed。Recovery 后可能发现该 batch CRC 合法并被重放，导致语义矛盾——调用方收到错误认为写入失败，但数据实际被恢复 + +**影响**: 调用方可能基于"写入失败"做非幂等业务决策（如放弃、走替代路径），但数据实际持久化了。 + +**建议修复**: + +拆分 WAL write failure 处理： + +```text +- encode 失败（未调用 write()）: 普通错误，write-stopped +- write() 失败（bytes 可能已交给 OS）: ErrCommitUnknown + write-stopped + 除非实现能证明零 bytes 到达文件（例如 write 返回 0 且无副作用） +``` + +--- + +### C2. Segment 边界跨 Batch 行为未定义 + +**严重程度**: Critical +**位置**: Section 3.2 "Block 边界处理"（~line 303）与 "Recovery 扫描流程"（~line 436） + +**问题描述**: + +WAL Batch 可拆成多个 Physical Record 跨多个 Block，但未定义 WAL Batch 是否可以跨 segment 文件。Recovery 中 incomplete fragment 的处理取决于是否位于"最后一个需要恢复的 segment"（~line 493）： + +- 如果 Batch 可以跨 segment：recovery 必须在 segment 间携带 fragment 收集状态（CollectingFragments），这增加了恢复复杂度 +- 如果 Batch 不可跨 segment：需要显式约束 segment rotation 时机 + +当前 recovery 流程按 segment 顺序独立扫描，未定义跨 segment fragment 收集。 + +**影响**: 可能导致合法的跨 segment batch 被误判为中间损坏，或需要引入复杂的跨 segment 状态管理。 + +**建议修复**: + +在 Section 3.2 明确添加约束： + +```text +WAL Batch 不得跨 segment 文件。Segment rotation 只在 WAL Batch 边界发生。 +当前 segment 写入完一个完整 WAL Batch 后，如果需要轮转，在下个 Batch 写入前切换到新 segment。 +``` + +--- + +### C3. 预创建 segment 可破坏尾部截断逻辑 + +**严重程度**: Critical +**位置**: Section 3.2 "WAL 元数据持久化协议"（~line 173）与 "Physical Record 解析规则"（~line 491） + +**问题描述**: + +新 segment 创建协议（步骤 1-8）允许 `segment-N+1.wal` 在文件系统上可见（已完成 rename + directory fsync），即使它尚未承载任何 batch。如果此时进程崩溃，`segment-N.wal` 可能有 crash-torn tail。 + +Recovery 扫描时，因为 `segment-N+1.wal` 存在且 header 合法，`segment-N` 不再被视为"最后一个需要恢复的 segment"。按照当前的尾部/中间损坏分类规则，`segment-N` 的尾部损坏会被升级为"WAL 中间损坏"→ 报错而非截断。 + +**影响**: 一个本应可截断恢复的尾部 partial write 场景被错误升级为不可恢复的中间损坏，导致整个 DB 无法启动。 + +**建议修复**: + +方案 A（推荐）：禁止预创建未来 segment，直到当前 segment 在完整 batch 边界 sealed： + +```text +新 segment 只在当前 active segment 写完一个完整 WAL Batch 后才创建。 +确保 segment-N 永远在完整 batch 边界结束，segment-N+1 的创建不先于该 sealing。 +``` + +方案 B：Recovery 能识别并忽略空 segment（startSequence == expectedSequence 但无任何 batch）： + +```text +如果 segment header 合法但不含任何 complete batch，且 startSequence == expectedSequence， +视为空 segment，跳过或删除，继续扫描下一个 segment。 +``` + +--- + +### C4. 恢复后未 fsync 确认的 Batch 可能变成已发布 + +**严重程度**: Critical +**位置**: Section 3.2 "恢复完成状态"（~line 575）与 "持久化策略"（~line 96） + +**问题描述**: + +Recovery 重放所有 CRC 合法、sequence 连续的 WAL batch，并在恢复完成后将其全部标记为 `published`。但其中可能包含崩溃前从未 fsync 确认（调用方未收到成功）的 batch。 + +场景： +1. WAL bytes 已通过 `write()` 写入 OS page cache +2. 进程崩溃（非掉电），OS 将 page cache 刷盘 +3. 重启后 recovery 发现该 batch 完整、CRC 合法、sequence 连续 +4. 该 batch 被重放并标记为 published +5. 但调用方从未收到成功确认 + +**影响**: `Always` 策略下，调用方收到的语义是 "Put 返回成功 = 已持久化"。但如果进程 crash（非掉电），未确认的写入可能变成已发布。这是一个 API 语义问题而非数据安全问题。 + +**建议修复**: + +在 Section 3.2 "持久化策略" 或 "恢复完成状态" 中显式声明： + +```text +进程崩溃（非掉电）后恢复时，OS page cache 中已写入但尚未 fsync 的完整 WAL Batch +可能被恢复并视为已发布。这不是数据丢失，而是数据可见性前移。 +调用方必须理解：进程崩溃重启后，比掉电场景可能多恢复一些写入。 + +如果需要严格区分"调用方已确认"与"未确认但存在于 WAL"，需要后续引入 +durable commit marker 或 confirmed-sequence 元数据。 +``` + +--- + +### C5. Go 内存模型：lock-free read 需要原子发布机制 + +**严重程度**: Critical +**位置**: Section 3.3 MemTable "并发策略"（~line 600）与 Section 3.2 步骤 ⑥⑧（~line 109-112） + +**问题描述**: + +Section 3.3 声明 MemTable 使用 "Mutex 写 + 无锁读"，Section 3.2 步骤 ⑥ 在 fsync 前将 entry 写入 MemTable（pending 状态），步骤 ⑧ 通过更新 `publishedSequence` 使 entry 对无锁读可见。 + +在 Go 内存模型中： +1. **Skiplist 节点发布**：Mutex 保护下的写入对未持锁的并发读者不一定可见。需要 `atomic.Pointer` 或等效发布机制确保节点对读者可见。 +2. **`publishedSequence` 更新**：作为普通变量写入，无锁读者可能看到过时值或部分写入。必须是 atomic 操作。 + +**影响**: 在 ARM 架构（弱内存序）上可能出现读者看到 `publishedSequence` 已更新但对应 skiplist 节点尚未可见的情况，导致读到不一致数据。 + +**建议修复**: + +在 Section 3.2 或 3.3 中明确内存序要求： + +```text +1. MemTable skiplist 节点必须通过 atomic store（atomic.Pointer 或自定义 release 操作）发布， + 确保无锁读者看到完整的节点内容。 +2. publishedSequence 必须是 atomic 变量（atomic.Uint64）， + 且其 Store 必须在所有 batch entries 的 skiplist 节点都已原子发布之后执行。 + 这保证读者先看到节点，再通过 publishedSequence 筛选可见 entry。 +3. 读者必须先 atomic Load publishedSequence，再遍历 skiplist。 +``` + +--- + +## Important Issues + +### I1. MemTable 写入失败（Arena 满）在 WAL 写入后未覆盖 + +**严重程度**: Important +**位置**: Section 3.2 "写入流程" 步骤 ⑥（~line 109） + +**问题描述**: + +写入流程步骤 ⑤（WAL encode/write）成功后，步骤 ⑥ 写入 MemTable 可能因为 Arena 满而失败。此时 WAL bytes 已持久化（或已在 page cache），但 MemTable 中没有对应 entry。设计文档未覆盖此场景。 + +**建议修复**: + +```text +方案 A（推荐）：在 WAL write 之前保证 MemTable 有足够容量。 + 写入前检查 Arena 剩余空间，不足时先冻结 MemTable 并创建新 MemTable。 + Arena 预留必须考虑最大可能的 batch size。 + +方案 B：MemTable 写入失败后按 ErrCommitUnknown + write-stopped 处理。 + 因为 WAL bytes 可能已持久化，不能按普通错误处理。 +``` + +--- + +### I2. `CURRENT` 文件权威性与实际恢复模型不一致 + +**严重程度**: Important +**位置**: Section 3.2 "CURRENT / MANIFEST 权威性"（~line 420）与 "WAL 元数据持久化协议"（~line 181） + +**问题描述**: + +设计明确声明 `CURRENT` 只是写入侧辅助文件，recovery 权威源是 `MANIFEST + 目录扫描`。但 durable-ready 协议要求在 segment 可承载写入前更新 `CURRENT` 并 fsync（步骤 6-7）。这意味着 `CURRENT` 更新是 batch 确认成功的前提之一，但 recovery 又不依赖它。 + +**建议修复**: + +选择一种并保持一致： + +```text +方案 A（推荐）：简化 durable-ready 协议，移除 CURRENT 更新作为 batch 确认前提。 + Recovery 通过 MANIFEST + 目录扫描发现 segment，CURRENT 仅作为写入侧快速定位优化。 + 新 segment 只需 rename + WAL directory fsync 即可进入 durable-ready。 + +方案 B：让 CURRENT 成为 recovery 的必要组件。 + 这样需要处理 CURRENT 损坏/缺失的 fallback，增加恢复复杂度。不推荐。 +``` + +--- + +### I3. 尾部截断后缺少持久化步骤 + +**严重程度**: Important +**位置**: Section 3.2 "Physical Record 解析规则" 尾部损坏处理（~line 495） + +**问题描述**: + +Recovery 允许截断最后一个 segment 的尾部损坏。但截断操作本身（`ftruncate` + 删除后续空 segment）需要 fsync 才能在再次崩溃时保持一致性。设计文档未说明截断后的持久化步骤。 + +**建议修复**: + +在 "恢复完成状态" 之后或 "Recovery 扫描流程" 末尾添加： + +```text +截断持久化步骤： +1. ftruncate active segment 到 lastCompleteBatchEnd +2. fsync truncated segment +3. 删除 startSequence == expectedSequence 但无 complete batch 的后续空 segment +4. fsync WAL directory +5. 更新 MANIFEST 记录恢复终点 +6. fsync metadata directory +以上完成后，引擎才能开始接受新写入。 +``` + +--- + +### I4. Batch 校验缺少资源上限 + +**严重程度**: Important +**位置**: Section 3.2 "WAL Batch 校验与重放"（~line 530） + +**问题描述**: + +Recovery 校验 `entryCount`、`entriesSize` 和 entry 边界，但未定义任何资源上限。恶意或损坏的 WAL 可能包含极大的 `entryCount` 或 `entriesSize`，导致 recovery OOM 或无限循环。 + +**建议修复**: + +在 Section 3.2 添加硬性限制： + +```text +WAL Batch 资源上限（可配置，建议默认值）： +- entryCount: 最大 10,000 +- entriesSize: 最大 4MB +- 单个 keyLen: 最大 4KB（不含 value） +- 单个 valLen (Inline): 最大 4KB（超过走 ValueLogPointer） +- fragment buffer: 最大 entriesSize 上限 +- varint: 最大 5 bytes（u64 varint 上限） + +Recovery 解析时，超过任何上限即视为 WAL 损坏。 +写入侧也必须遵守这些限制，超出拒绝写入。 +``` + +--- + +### I5. `publishedSequence` 需要明确的内存序约束 + +**严重程度**: Important +**位置**: Section 3.2 "可见性语义"（~line 148） + +**问题描述**: + +`publishedSequence` 作为普通变量描述其语义，但未说明其在 Go 内存模型中的操作类型。多 goroutine 并发读写需要明确的 happens-before 关系。 + +**建议修复**: + +在 "可见性语义" 小节补充： + +```text +publishedSequence 的内存序约束： +1. 类型：atomic.Uint64（或等效原子变量） +2. 写入侧：Store 只在 WAL durability 和所有 MemTable 节点原子发布都完成后执行 +3. 读取侧：Load 在遍历 MemTable 前执行，获得可见性 high-water mark +4. Happens-before 关系： + WAL fsync 完成 → MemTable 节点原子发布 → publishedSequence.Store + → 读者 publishedSequence.Load → 遍历 MemTable 筛选可见 entry +``` + +--- + +## 变更追踪 + +| Issue | 类型 | 优先级 | 状态 | +|-------|------|--------|------| +| C1 | 语义正确性 | Critical | Open | +| C2 | 格式完整性 | Critical | Open | +| C3 | 恢复正确性 | Critical | Open | +| C4 | API 语义 | Critical | Open | +| C5 | 内存安全 | Critical | Open | +| I1 | 错误处理完整性 | Important | Open | +| I2 | 设计一致性 | Important | Open | +| I3 | 持久化完整性 | Important | Open | +| I4 | 安全性/鲁棒性 | Important | Open | +| I5 | 内存序正确性 | Important | Open | diff --git a/docs/phase1-wal-plan.md b/docs/phase1-wal-plan.md new file mode 100644 index 0000000..40f4384 --- /dev/null +++ b/docs/phase1-wal-plan.md @@ -0,0 +1,640 @@ +# Phase 1: WAL 子系统开发方案 + +基于 `docs/design.md` §3.2 设计文档。 + +## 目标 + +实现完整的 WAL（预写日志）子系统，使其能够支撑单 key autocommit 的写入、崩溃恢复和读可见性语义。 + +## 开发阶段总览 + +``` +Phase 1A: 项目骨架 + WAL 编码格式层 +Phase 1B: WAL 文件写入 + Segment 管理 +Phase 1C: WAL Writer（Group Commit） +Phase 1D: WAL Recovery +Phase 1E: MemTable（SkipList + Arena） +Phase 1F: 写入路径集成（WAL → MemTable 完整流水线） +Phase 1G: 读路径 + 嵌入式 API +Phase 1H: MANIFEST + 文件管理 +Phase 1I: 集成测试 + Benchmark +``` + +--- + +## Phase 1A: 项目骨架 + WAL 编码格式层 + +**目标**: 建立 Go 项目结构，实现 WAL 物理格式（Block / Physical Record / WAL Batch / Entry）的编码与解码。 + +### 任务 + +#### 1A-1: 项目初始化 +- `go.mod` 初始化（模块名 `github.com/dailz/go-kv`） +- 目录结构： + ``` + go-kv/ + ├── go.mod + ├── wal/ # WAL 子系统 + │ ├── wal.go # 公共类型、常量、配置 + │ ├── record.go # Physical Record 编解码 + │ ├── batch.go # WAL Batch 编解码 + │ ├── entry.go # Entry 编解码 + │ ├── header.go # WAL File Header 编解码 + │ └── wal_test.go + ├── memtable/ # MemTable（Phase 1E） + ├── config/ # 全局配置 + ├── errors.go # 公共错误类型 + └── db.go # DB 入口 + ``` +- `.golangci.yml` 配置（参考 golang-lint skill） + +#### 1A-2: 公共错误类型 (`errors.go`) +- `ErrCommitUnknown` — maybe committed 语义 +- `ErrWriteStopped` — 引擎 write-stopped +- `ErrSequenceExhausted` — sequence 耗尽 +- `ErrWALCorrupted` — WAL 损坏 +- `ErrInvalidConfig` — 配置不合法 + +#### 1A-3: WAL 常量与配置 (`wal/wal.go`) +```go +const ( + WalMagic uint32 = 0x... // 待定 + WalFormatVersion uint16 = 1 + WalFileHeaderSize = 32 + WalBlockSize = 32 * 1024 // 32KB + PhysicalRecordHeaderSize = 7 + WalBatchHeaderSize = 18 + MaxWalBatchEntryCount = 10_000 + MaxWalBatchEntriesSize = 4 * 1024 * 1024 // 4MB + MaxWalKeyBytes = 4 * 1024 // 4KB + MaxWalInlineValueBytes = 4 * 1024 // 4KB + MaxWalVarintBytes = 5 + DefaultMaxWalSegmentSize = 64 * 1024 * 1024 // 64MB + DefaultImmutableCount = 2 +) + +// Fragment types +const ( + RecInvalid uint8 = 0 + RecFull uint8 = 1 + RecFirst uint8 = 2 + RecMiddle uint8 = 3 + RecLast uint8 = 4 +) + +// OpType +const ( + OpInvalid uint8 = 0 + OpPut uint8 = 1 + OpDelete uint8 = 2 +) + +// ValueKind +const ( + VKNone uint8 = 0 + VKInline uint8 = 1 + VKValueLogPointer uint8 = 2 +) +``` + +WAL 配置结构体： +```go +type WalConfig struct { + MaxSegmentSize uint64 // default 64MB + BlockSize uint32 // default 32KB + SyncMode SyncMode // Always/Periodic/Never + PeriodicSyncMs uint32 // Periodic 模式的 fsync 间隔 + MaxBatchEntries uint32 // default 10000 + MaxBatchSize uint32 // default 4MB + MaxKeyBytes uint32 // default 4KB + MaxInlineValue uint32 // default 4KB +} +``` + +配置校验函数 — 必须在 DB 打开时验证不变量： +```text +maxWalSegmentPayload >= maxEncodedWalBatchSize + worstCasePhysicalRecordOverhead + worstCaseBlockPadding +``` + +#### 1A-4: WAL File Header 编解码 (`wal/header.go`) +- `WalFileHeader` 结构体：magic, formatVersion, headerSize, blockSize, segmentID, startSequence, headerCRC +- `EncodeWalHeader(h *WalFileHeader) [WalFileHeaderSize]byte` +- `DecodeWalHeader(data []byte) (*WalFileHeader, error)` — 校验 magic、formatVersion、headerSize、headerCRC +- CRC 覆盖范围：magic 到 startSequence，不包含 headerCRC 自身 +- 字节序：little-endian + +#### 1A-5: Physical Record 编解码 (`wal/record.go`) +- `PhysicalRecord` 结构体：CRC, Length, Type, Payload +- `EncodePhysicalRecord(recType uint8, payload []byte) []byte` — 返回编码后的 bytes +- `DecodePhysicalRecord(data []byte) (*PhysicalRecord, error)` — CRC 校验 +- Block 边界处理辅助函数： + - `PaddingNeeded(blockOffset, blockSize uint32) int` — 剩余空间 <= 7 时返回需要 padding 的字节数 + - `CanFitRecord(blockOffset, blockSize, payloadLen uint32) bool` + +#### 1A-6: WAL Batch 编解码 (`wal/batch.go`) +- `WalBatch` 结构体：Flags, BaseSequence, EntryCount, EntriesSize, Entries +- `EncodeWalBatch(batch *WalBatch) ([]byte, error)` — 编码 Batch Header + Entries +- `DecodeWalBatch(data []byte) (*WalBatch, error)` — 校验 flags、entryCount、entriesSize +- Batch 分片：`SplitIntoRecords(encodedBatch []byte, blockSize uint32) [][]byte` — 将编码后的 Batch 拆分为 Physical Record payloads +- Batch 重组：`FragmentCollector` — 收集 fragments 并重组成完整 Batch + +FragmentCollector 状态机： +``` +Idle → 收到 Full → 重放 batch → Idle +Idle → 收到 First → CollectingFragments +CollectingFragments → 收到 Middle → 追加 +CollectingFragments → 收到 Last → 重组 → 重放 → Idle +``` + +#### 1A-7: Entry 编解码 (`wal/entry.go`) +- `WalEntry` 结构体：OpType, ValueKind, Key, Value +- `EncodeEntry(e *WalEntry) ([]byte, error)` — 编码为 varint 长度 + bytes +- `DecodeEntry(data []byte) (*WalEntry, int, error)` — 解码，返回 entry 和 consumed bytes +- 校验规则： + - keyLen > 0 && keyLen <= maxKeyBytes + - Put 要求 valueKind ∈ {Inline, ValueLogPointer} + - Put + Inline: valLen <= maxInlineValueBytes (允许 valLen = 0) + - Put + ValueLogPointer: valLen > 0 + - Delete: valueKind == None, valLen == 0 + +#### 1A-8: WAL Batch 资源校验 +- `ValidateBatchLimits(entries []*WalEntry) error` — 在 sequence 分配之前检查： + - entryCount <= maxBatchEntries + - 每个 keyLen <= maxKeyBytes + - 每个 inline valLen <= maxInlineValueBytes + - entries 编码后总大小 <= maxBatchSize + - 单个 Batch 的最坏 Physical Record overhead 不超过 segment capacity + +### 验收标准 +- [ ] 所有编解码函数有 table-driven test +- [ ] CRC 校验正确 +- [ ] Fragment 分片/重组 round-trip 正确 +- [ ] 资源限制校验覆盖所有边界条件 +- [ ] `go vet` / `golangci-lint` 通过 + +--- + +## Phase 1B: WAL 文件写入 + Segment 管理 + +**目标**: 实现 WAL segment 文件的写入、轮转和持久化协议。 + +### 任务 + +#### 1B-1: Segment 文件格式写入器 (`wal/segment_writer.go`) +- `SegmentWriter` — 封装 WAL segment 文件的追加写入 +- 状态：当前 segment fd、当前 block offset、当前 segmentID、payload written bytes +- `NewSegmentWriter(dir string, segmentID uint64, startSequence uint64, cfg *WalConfig) (*SegmentWriter, error)` + - 创建 segment-N.wal.tmp + - 写入 WAL File Header + - fsync + - rename → segment-N.wal + - fsync directory + - 进入 durable-ready 状态 +- `AppendBatch(batch *WalBatch) error` — 编码 batch → split into records → 按 block 边界写入 +- `Sync() error` — fsync 当前 segment 文件 +- `Close() error` +- `RemainingPayload() uint64` — 当前 segment 剩余可用 payload 空间 +- `CurrentOffset() uint64` — 当前写入偏移 + +#### 1B-2: Block 写入缓冲 (`wal/block_writer.go`) +- 管理 32KB block 的填充和 padding +- `BlockWriter` — 封装 block 内的 Physical Record 写入 +- 自动处理 block 边界：剩余 <= 7 bytes 时 padding +- 跨 block 的 batch fragment 自动拆分 + +#### 1B-3: Segment 轮转逻辑 +- 写入 batch 前检查 `RemainingPayload()` 是否足够容纳整个 batch +- 不足时：当前 segment 完成（在 batch 边界）、创建新 segment +- 新 segment 的 durable-ready 协议： + 1. create segment-N+1.wal.tmp + 2. write WAL File Header（含 startSequence = nextExpectedSequence） + 3. fsync segment-N+1.wal.tmp + 4. rename → segment-N+1.wal + 5. fsync WAL directory + 6. segment-N+1 进入 durable-ready +- 旧的 active segment 密封 + +#### 1B-4: CURRENT 文件管理 +- best-effort 更新 CURRENT 文件 +- temp + rename 模式 +- 更新失败不影响已 durable-ready 的 segment + +#### 1B-5: WAL 目录管理工具 +- 扫描 WAL 目录中的 segment 文件 +- 按 segmentID 排序 +- 解析文件名中的 segmentID +- 文件名格式：`segment-{id}.wal` + +### 验收标准 +- [ ] Segment 创建遵循 durable-ready 协议 +- [ ] Batch 不跨 segment +- [ ] Block padding 正确 +- [ ] Segment 轮转在 batch 边界发生 +- [ ] 多 segment 写入后，每个 segment 的 header 可以正确解析 +- [ ] 测试覆盖：正常写入、跨 block batch、segment 轮转触发 + +--- + +## Phase 1C: WAL Writer（Group Commit） + +**目标**: 实现完整的 WAL 写入路径，包括 group commit、sequence 管理、fsync 策略和错误分类。 + +### 任务 + +#### 1C-1: Sequence 管理器 (`wal/sequence.go`) +- `SequenceManager` — 管理 WAL 物理 mutation sequence +- `atomic.Uint64` 存储 nextSequence、publishedSequence、durableSequence +- `AllocateBatch(count uint32) (baseSequence uint64, err error)` — checked arithmetic 检查溢出 +- `Publish(sequence uint64)` — release 语义 store publishedSequence +- `MarkDurable(snapshot SegmentEndState)` — 推进 durableSequence +- `Published() uint64` — load publishedSequence +- `Durable() uint64` — load durableSequence + +#### 1C-2: Commit Queue (`wal/commit_queue.go`) +- 写请求进入的队列 +- 每个写请求关联一个 `*sync.Cond` 或 channel 用于等待/唤醒 +- `CommitBatch` 结构体：entries、完成 channel、错误结果、baseSequence + +#### 1C-3: WAL Writer 主循环 (`wal/writer.go`) +核心写入循环： +``` +loop: + 1. 从 commit queue 收集一批写入 + 2. 等待触发条件（500µs 或 32KB）或 queue 非空 + 3. 组装 WAL Batch + 4. 校验 batch 资源限制 + 5. 预留 MemTable Arena 容量 + 6. 分配 sequence（baseSequence） + 7. 在私有缓冲区编码 WAL Batch + 8. 检查/触发 segment 轮转 + 9. Append WAL Batch 到 segment 文件 + 10. 写入 MemTable（pending/unpublished） + 11. fsync（Always 模式） + 12. 发布 publishedSequence + 13. 唤醒所有等待的调用方 +``` + +错误分类逻辑： +- 步骤 4-7 失败（未分配 sequence）→ 普通错误，可继续 +- 步骤 6 后失败（sequence 已分配）→ write-stopped +- 步骤 9 后失败（WAL write 已尝试）→ ErrCommitUnknown + write-stopped +- 步骤 11 失败（fsync）→ ErrCommitUnknown + write-stopped + +#### 1C-4: Fsync 策略实现 (`wal/fsync.go`) +- `SyncMode` 类型：Always / Periodic / Never +- `Always`: 每次 batch fsync 后再 publish +- `Periodic`: 后台 goroutine 定期 fsync，write 成功即可 publish +- `Never`: 不主动 fsync +- `Periodic` 的 fsync worker： + - 快照当前 append high-water mark: (segmentID, endOffset, endSequence) + - fsync 成功后按连续 batch 推进 durableSequence + - fsync 失败 → write-stopped + +#### 1C-5: durableSequence 推进逻辑 +- 每个 batch 记录 `(segmentID, endOffset, endSequence)` +- fsync snapshot 后只推进满足条件的最大连续 batch +- 跨 segment 推进需要 segment 已 durable-ready + +#### 1C-6: Write-Stopped 状态管理 +- `atomic.Bool` 存储 writeStopped +- 进入 write-stopped 后拒绝新写入 +- 已存在的 MemTable / Immutable MemTable 可继续后台处理 +- 提供 `IsWriteStopped() bool` 查询接口 + +### 验收标准 +- [ ] Group commit 正确合并多个写请求 +- [ ] 双触发（时间/大小）工作正常 +- [ ] Sequence 分配无溢出 +- [ ] Always 模式下 publish 在 fsync 之后 +- [ ] 错误分类准确（普通错误 / write-stopped / ErrCommitUnknown） +- [ ] 并发写入正确（多 goroutine 同时 Put） +- [ ] Write-stopped 后新写入被拒绝 + +--- + +## Phase 1D: WAL Recovery + +**目标**: 实现 WAL 崩溃恢复，包括 segment 扫描、fragment 重组、batch 校验和尾部截断。 + +### 任务 + +#### 1D-1: Segment 扫描器 (`wal/scanner.go`) +- 从 WAL 目录扫描 segment 文件 +- 按 segmentID 排序 +- 从 MANIFEST 指定的 recoverySegmentID 开始 +- 过滤掉 segmentID < recoverySegmentID 的旧 segment +- 校验连续性：segmentID 和 startSequence 都必须连续 + +#### 1D-2: Physical Record 解析器 (`wal/record_parser.go`) +- Block 级别的顺序解析 +- 处理 padding（全 0 校验） +- Physical Record header 解析和 CRC 校验 +- 错误分类：尾部 vs 中间损坏 + +#### 1D-3: Fragment 重组器 (`wal/fragment_collector.go`) +- 实现 Idle / CollectingFragments 状态机 +- 收集 First / Middle / Last fragments +- Buffer 大小限制（Batch Header 长度 + entriesSize 上限） +- Fragment 顺序合法性检查 + +#### 1D-4: Batch 校验与重放 (`wal/recovery.go`) +- Batch Header 校验：flags、entryCount、entriesSize +- Batch sequence 连续性：batch.baseSequence == expectedSequence +- Entry 逐条校验：opType、valueKind、keyLen、valLen +- 重放回调：对每个合法 entry 调用 replay 函数 +- Sequence 推进：expectedSequence += entryCount + +#### 1D-5: 尾部截断持久化 (`wal/truncation.go`) +- 识别最后一个完整 batch 的结束位置 +- ftruncate segment 文件 +- fsync 被截断的 segment +- 删除不含任何 complete batch 的后续空 segment +- fsync WAL directory +- 任一步失败 → recovery 报错 + +#### 1D-6: Recovery 主流程 (`wal/recovery.go`) +``` +1. 读取 MANIFEST → recoverySegmentID +2. 扫描 WAL 目录 → 过滤出 recovery segments +3. 排序并校验连续性 +4. 逐 segment 扫描： + a. 校验 File Header + b. 逐 Block 解析 Physical Records + c. Fragment 重组 → 完整 Batch + d. Batch 校验 → 重放 + e. 更新 expectedSequence +5. 处理尾部异常 +6. 持久化截断（如需要） +7. 返回恢复结果：recoveredSequence, nextSequence, publishedSequence +``` + +### 验收标准 +- [ ] 正常 WAL 完整恢复 +- [ ] 尾部 partial write 正确截断 +- [ ] 中间损坏正确报错 +- [ ] 跨 segment 恢复正确 +- [ ] Fragment 重组 round-trip 正确 +- [ ] Segment 连续性校验 +- [ ] Sequence 溢出检测 +- [ ] 资源限制校验（recovery 侧） + +--- + +## Phase 1E: MemTable（SkipList + Arena） + +**目标**: 实现基于 Arena 的 SkipList，支持 pending/unpublished/aborted 状态，容量预留，和原子发布。 + +### 任务 + +#### 1E-1: Arena 分配器 (`memtable/arena.go`) +- 固定大小 Arena（默认 64MB） +- 线程安全的内存分配 +- 对齐分配 +- 剩余容量查询 +- 支持预留（reserve）操作 + +#### 1E-2: SkipList (`memtable/skiplist.go`) +- 最大 20 层 +- Mutex 写 + 无锁读 +- `atomic.Pointer` 发布 next 指针（release 语义） +- 有序遍历（Iterator） +- key 比较（bytes comparison） + +#### 1E-3: Entry 状态管理 (`memtable/entry.go`) +- Entry 结构：key、value、sequence、pending/aborted 标记 +- 原子发布：`atomic.Pointer` store-release +- 可见性判断：`entry.sequence <= loadedPublishedSequence && !aborted` + +#### 1E-4: MemTable (`memtable/memtable.go`) +- 封装 SkipList + Arena +- `Put(key, value, sequence) error` — 写入 pending entry +- `PublishEntries(upToSequence)` — 批量发布 pending entries +- `AbortEntries(fromSequence)` — 标记 aborted +- `Get(key, publishedSequence) (GetResult, error)` — 无锁读，只返回 sequence <= publishedSequence 且非 aborted 的 entry +- `NewIterator(publishedSequence) Iterator` — 无锁有序遍历 +- `ApproximateSize() uint64` — 近似内存使用量 +- `IsFull() bool` +- `Reserve(entries []ReserveEntry) (uint64, error)` — 容量预留（最坏情况计算） + +#### 1E-5: 容量预留计算 +- 每个 entry 的预留大小 = key bytes + value bytes + skiplist node overhead + next 指针数组（最大层高）+ arena 对齐 padding +- 批量预留必须覆盖整个 batch +- checked arithmetic 检查单个 batch 是否超过空 MemTable 容量 + +#### 1E-6: Immutable MemTable 管理 +- Freeze 流程：当前 MemTable → Immutable +- Immutable 队列（上限 2） +- 队列满时阻塞 + +### 验收标准 +- [ ] SkipList 正确性：插入、查找、有序遍历 +- [ ] Arena 分配无泄漏 +- [ ] 并发读写正确（racetest） +- [ ] pending/unpublished entry 对读不可见 +- [ ] 发布后 entry 可见 +- [ ] aborted entry 对读不可见 +- [ ] 容量预留准确 +- [ ] 内存序正确（go test -race 通过） + +--- + +## Phase 1F: 写入路径集成 + +**目标**: 将 WAL Writer 和 MemTable 连通，实现完整的写入流水线。 + +### 任务 + +#### 1F-1: DB 写入 API (`db.go`) +```go +type DB struct { ... } + +func Open(dir string, opts ...Option) (*DB, error) +func (db *DB) Close() error +func (db *DB) Put(key, value []byte) error +func (db *DB) Delete(key []byte) error +``` + +#### 1F-2: 写入路径集成 +完整写入流程： +``` +Put(key, value) + → commit queue + → group commit 组装 batch + → 校验 batch limits + → 预留 MemTable Arena + → 分配 sequence + → 私有缓冲编码 + → 检查 segment 轮转 + → WAL append + → MemTable pending write + → fsync（Always 模式） + → 原子发布 MemTable entries + → 推进 publishedSequence + → 唤醒调用方 +``` + +#### 1F-3: MemTable Freeze + Switch +- 写入前检查容量，不足时 freeze + switch +- Immutable 队列满时阻塞等待 +- Freeze 时确保当前 MemTable 已完成所有 pending 发布 + +#### 1F-4: 恢复启动集成 +- Open 时执行 recovery +- 恢复的 entries 写入 MemTable 并标记为 published +- 设置 nextSequence、publishedSequence + +### 验收标准 +- [ ] 单条 Put 写入成功 +- [ ] 并发 Put 正确 +- [ ] 写入后读取可见（Always 模式） +- [ ] WAL crash recovery 后数据完整 +- [ ] MemTable freeze/switch 正确 +- [ ] Sequence 连续无间隙 +- [ ] `go test -race` 通过 + +--- + +## Phase 1G: 读路径 + 嵌入式 API + +**目标**: 实现完整的读路径和嵌入式 API。 + +### 任务 + +#### 1G-1: Get API +```go +type GetResult struct { + Value []byte + Found bool +} + +func (db *DB) Get(key []byte) (GetResult, error) +``` + +#### 1G-2: 读路径实现 +- 读取 publishedSequence（atomic load） +- 遍历 MemTable → Immutable MemTables +- 只返回 `sequence <= publishedSequence` 且非 aborted 的 entry +- Delete (tombstone) 返回 `Found=false` + +#### 1G-3: 辅助 API +```go +func (db *DB) GetDurableSequence() uint64 +func (db *DB) IsWriteStopped() bool +``` + +### 验收标准 +- [ ] Put 后 Get 返回正确值 +- [ ] Delete 后 Get 返回 Found=false +- [ ] 空 value 正确区分（Found=true, Value=[]byte{}） +- [ ] 并发读写正确 +- [ ] 未发布 entry 对 Get 不可见 + +--- + +## Phase 1H: MANIFEST + 文件管理 + +**目标**: 实现 MANIFEST 持久化和 WAL segment 生命周期管理。 + +### 任务 + +#### 1H-1: MANIFEST 格式 +- 记录 recovery 起点的 recoverySegmentID +- temp + rename 原子更新 +- MANIFEST 只在 checkpoint（MemTable flush）后推进 + +#### 1H-2: 首次创建 DB 流程 +- 创建目录结构 +- 创建初始 MANIFEST（recoverySegmentID=0） +- 创建初始 WAL segment + +#### 1H-3: WAL Segment 生命周期 +- 旧 segment 删除条件：已被 MANIFEST checkpoint 覆盖 +- 删除顺序：先删除文件，再 fsync directory + +### 验收标准 +- [ ] 首次创建 DB 成功 +- [ ] 重复打开 DB 正确恢复 +- [ ] MANIFEST 原子更新 +- [ ] 旧 WAL segment 正确清理 + +--- + +## Phase 1I: 集成测试 + Benchmark + +**目标**: 端到端测试和性能基准。 + +### 任务 + +#### 1I-1: 集成测试 +- 正常写入 + 读取 round-trip +- 并发写入 + 读取一致性 +- 崩溃恢复（kill -9 模拟） +- WAL 尾部损坏恢复 +- Write-stopped 后的行为 +- Sequence 耗尽处理 +- 配置校验拒绝非法配置 +- 空 value 写入/读取 +- 大量数据写入（触发 segment 轮转） + +#### 1I-2: Benchmark +- 单线程 Put 吞吐 +- 多线程 Put 吞吐 +- 单线程 Get 延迟 +- 多线程 Get 延迟 +- WAL Recovery 时间 +- 写入放大测量 + +#### 1I-3: Race Condition 测试 +- `go test -race -count=100` +- 并发 Put + Get +- 并发 Put + Close + +### 验收标准 +- [ ] 所有集成测试通过 +- [ ] Benchmark 数字可作为后续优化基线 +- [ ] Race test 无 data race + +--- + +## 依赖关系与并行度 + +``` +1A ─────┐ + ├── 1B ─────┐ + │ ├── 1C ─────┐ + │ │ ├── 1F ── 1G ── 1I + │ │ │ +1A ── 1E ──────────┘ │ + │ │ + ├── 1D ─────────────────┘ + │ + └── 1H ────────────────────────────── 1I +``` + +可并行开发的模块： +- 1A 完成后，1B/1D/1E/1H 可以并行开发 +- 1B 完成后，1C 可以开始 +- 1C + 1D + 1E 完成后，1F 可以集成 +- 1F + 1H 完成后，1G 可以集成 +- 所有完成后，1I 集成测试 + +## 技术要点备忘 + +### 内存序（最关键） +- skiplist next 指针：`atomic.Pointer` store-release +- `publishedSequence`：`atomic.Uint64` store（在所有 entry 节点发布后） +- 读者先 load publishedSequence（acquire），再遍历 skiplist + +### WAL 副作用边界 +- 未调用 `write()` → 普通错误 +- 已调用 `write()` → ErrCommitUnknown + write-stopped +- 私有缓冲区编码，不共享 bufio.Writer + +### ErrCommitUnknown 语义 +- maybe committed，不是 definitely failed +- 不盲目重试 +- 第一阶段为弱确认 + +### WAL Batch 资源前置校验 +- sequence 分配之前完成所有可失败校验 +- 减少 write-stopped 触发机会 diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..461c8ce --- /dev/null +++ b/errors.go @@ -0,0 +1,22 @@ +package go_kv + +import "errors" + +// ErrCommitUnknown indicates the commit result is indeterminate: the WAL write +// was attempted but the caller cannot assume the write definitely failed or +// succeeded. The caller must query the commit state before retrying. +var ErrCommitUnknown = errors.New("go-kv: commit result unknown") + +// ErrWriteStopped indicates the engine has entered a write-stopped state and +// rejects all subsequent writes. +var ErrWriteStopped = errors.New("go-kv: write stopped") + +// ErrSequenceExhausted indicates the sequence number space is exhausted. The +// engine enters a terminal state requiring database migration or rebuild. +var ErrSequenceExhausted = errors.New("go-kv: sequence exhausted") + +// ErrWALCorrupted indicates WAL data corruption was detected. +var ErrWALCorrupted = errors.New("go-kv: WAL corrupted") + +// ErrInvalidConfig indicates an invalid configuration was provided. +var ErrInvalidConfig = errors.New("go-kv: invalid config") diff --git a/errors_test.go b/errors_test.go new file mode 100644 index 0000000..53fbc8d --- /dev/null +++ b/errors_test.go @@ -0,0 +1,42 @@ +package go_kv + +import ( + "errors" + "fmt" + "testing" +) + +func TestErrorTypes(t *testing.T) { + allErrors := []error{ + ErrCommitUnknown, + ErrWriteStopped, + ErrSequenceExhausted, + ErrWALCorrupted, + ErrInvalidConfig, + } + + // Each error must match itself via errors.Is. + for _, err := range allErrors { + if !errors.Is(err, err) { + t.Errorf("errors.Is(%v, %v) = false, want true", err, err) + } + } + + // Each error must NOT match any other error. + for i, a := range allErrors { + for j, b := range allErrors { + if i == j { + continue + } + if errors.Is(a, b) { + t.Errorf("errors.Is(%v, %v) = true, want false (distinct errors)", a, b) + } + } + } + + // Wrapped errors must still be identifiable via errors.Is. + wrapped := fmt.Errorf("operation failed: %w", ErrCommitUnknown) + if !errors.Is(wrapped, ErrCommitUnknown) { + t.Errorf("errors.Is(wrapped ErrCommitUnknown, ErrCommitUnknown) = false, want true") + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5363341 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/dailz/go-kv + +go 1.26.3 + +require github.com/stretchr/testify v1.11.1 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c4c1710 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/manifest/current.go b/manifest/current.go new file mode 100644 index 0000000..9f8b127 --- /dev/null +++ b/manifest/current.go @@ -0,0 +1,43 @@ +package manifest + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// WriteCurrent writes the CURRENT file to dir with a best-effort atomic rename. +// The file contains the active WAL segment filename (e.g. "segment-5.wal"). +// CURRENT is only a write-side hint; it may be missing or stale after a crash. +func WriteCurrent(dir string, segmentID uint64) error { + content := fmt.Sprintf("segment-%d.wal\n", segmentID) + tmpPath := dir + "/CURRENT.tmp" + if err := os.WriteFile(tmpPath, []byte(content), 0o644); err != nil { + return fmt.Errorf("write current tmp: %w", err) + } + if err := os.Rename(tmpPath, dir+"/CURRENT"); err != nil { + return fmt.Errorf("rename current: %w", err) + } + return nil +} + +// ReadCurrent reads the CURRENT file from dir and returns the segment ID. +// If the file does not exist or cannot be parsed, it returns 0, false with no error. +func ReadCurrent(dir string) (segmentID uint64, ok bool) { + data, err := os.ReadFile(dir + "/CURRENT") + if err != nil { + return 0, false + } + line := strings.TrimSpace(string(data)) + // Expected format: "segment-N.wal" + if !strings.HasPrefix(line, "segment-") || !strings.HasSuffix(line, ".wal") { + return 0, false + } + idStr := strings.TrimSuffix(strings.TrimPrefix(line, "segment-"), ".wal") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + return 0, false + } + return id, true +} diff --git a/manifest/doc.go b/manifest/doc.go new file mode 100644 index 0000000..66a1c6c --- /dev/null +++ b/manifest/doc.go @@ -0,0 +1,2 @@ +// Package manifest manages database metadata and checkpoint information. +package manifest diff --git a/manifest/manifest.go b/manifest/manifest.go new file mode 100644 index 0000000..3efa87f --- /dev/null +++ b/manifest/manifest.go @@ -0,0 +1,52 @@ +package manifest + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// Manifest holds database metadata used for recovery. +// The MANIFEST file stores the recovery checkpoint so that +// recovery knows which segments are already confirmed durable. +type Manifest struct { + RecoverySegmentID uint64 +} + +// Load reads the MANIFEST file from dir. +// If the file does not exist, it returns a zero-value Manifest with no error (fresh DB). +func Load(dir string) (*Manifest, error) { + path := dir + "/MANIFEST" + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &Manifest{RecoverySegmentID: 0}, nil + } + return nil, fmt.Errorf("read manifest: %w", err) + } + + line := strings.TrimSpace(string(data)) + if !strings.HasPrefix(line, "recovery_segment_id:") { + return nil, fmt.Errorf("manifest: invalid format: %q", line) + } + idStr := strings.TrimPrefix(line, "recovery_segment_id:") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("manifest: parse recovery_segment_id: %w", err) + } + return &Manifest{RecoverySegmentID: id}, nil +} + +// Save atomically writes the MANIFEST file to dir with the given recoverySegmentID. +func Save(dir string, recoverySegmentID uint64) error { + content := fmt.Sprintf("recovery_segment_id:%d\n", recoverySegmentID) + tmpPath := dir + "/MANIFEST.tmp" + if err := os.WriteFile(tmpPath, []byte(content), 0o644); err != nil { + return fmt.Errorf("write manifest tmp: %w", err) + } + if err := os.Rename(tmpPath, dir+"/MANIFEST"); err != nil { + return fmt.Errorf("rename manifest: %w", err) + } + return nil +} diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go new file mode 100644 index 0000000..e7ad0ab --- /dev/null +++ b/manifest/manifest_test.go @@ -0,0 +1,45 @@ +package manifest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestManifestRoundtrip(t *testing.T) { + dir := t.TempDir() + + err := Save(dir, 42) + require.NoError(t, err) + + m, err := Load(dir) + require.NoError(t, err) + assert.Equal(t, uint64(42), m.RecoverySegmentID) +} + +func TestManifestMissing(t *testing.T) { + dir := t.TempDir() + + m, err := Load(dir) + require.NoError(t, err) + assert.Equal(t, uint64(0), m.RecoverySegmentID) +} + +func TestCurrentRoundtrip(t *testing.T) { + dir := t.TempDir() + + err := WriteCurrent(dir, 5) + require.NoError(t, err) + + segmentID, ok := ReadCurrent(dir) + assert.True(t, ok) + assert.Equal(t, uint64(5), segmentID) +} + +func TestCurrentMissing(t *testing.T) { + dir := t.TempDir() + + _, ok := ReadCurrent(dir) + assert.False(t, ok) +} diff --git a/memtable/doc.go b/memtable/doc.go new file mode 100644 index 0000000..778d15c --- /dev/null +++ b/memtable/doc.go @@ -0,0 +1,2 @@ +// Package memtable implements an in-memory sorted key-value store. +package memtable diff --git a/wal/constants.go b/wal/constants.go new file mode 100644 index 0000000..b9eefa8 --- /dev/null +++ b/wal/constants.go @@ -0,0 +1,80 @@ +package wal + +// WAL file format constants. + +const ( + // WalMagic is the file type identifier for WAL segment files ("WALK"). + WalMagic uint32 = 0x57414C4B + + // WalFormatVersion is the WAL file format version. First version is 1. + WalFormatVersion uint16 = 1 + + // WalFileHeaderSize is the size of the WAL file header in bytes. + // Fields: magic(4) + formatVersion(2) + headerSize(2) + blockSize(4) + + // segmentID(8) + startSequence(8) + headerCRC(4) = 32. + WalFileHeaderSize = 32 + + // WalBlockSize is the fixed size of each WAL block in bytes (32 KB). + WalBlockSize = 32 * 1024 + + // PhysicalRecordHeaderSize is the size of a physical record header in bytes. + // Fields: crc32c(4) + length(2) + type(1) = 7. + PhysicalRecordHeaderSize = 7 + + // WalBatchHeaderSize is the size of a WAL batch header in bytes. + // Fields: flags(2) + baseSequence(8) + entryCount(4) + entriesSize(4) = 18. + WalBatchHeaderSize = 18 + + // MaxWalBatchEntryCount limits the number of entries in a single batch. + MaxWalBatchEntryCount uint32 = 10000 + + // MaxWalBatchEntriesSize limits the total size of the entries region in bytes (4 MB). + MaxWalBatchEntriesSize uint32 = 4 * 1024 * 1024 + + // MaxWalKeyBytes limits the size of a single key in bytes (4 KB). + MaxWalKeyBytes uint32 = 4 * 1024 + + // MaxWalInlineValueBytes limits the size of an inline value in bytes (4 KB). + // Values exceeding this must use ValueLogPointer. + MaxWalInlineValueBytes uint32 = 4 * 1024 + + // MaxWalVarintBytes is the maximum encoded length of a varint field. + MaxWalVarintBytes = 5 + + // DefaultMaxWalSegmentSize is the default maximum size of a WAL segment file (64 MB). + DefaultMaxWalSegmentSize uint64 = 64 * 1024 * 1024 +) + +// Fragment types for physical records. +const ( + // RecInvalid is an illegal fragment type used for corruption detection. + RecInvalid uint8 = 0 + // RecFull indicates a complete WAL batch in a single physical record. + RecFull uint8 = 1 + // RecFirst is the first fragment of a multi-record WAL batch. + RecFirst uint8 = 2 + // RecMiddle is a middle fragment (may appear zero or more times). + RecMiddle uint8 = 3 + // RecLast is the last fragment of a multi-record WAL batch. + RecLast uint8 = 4 +) + +// OpType represents the operation type of a WAL entry. +const ( + // OpInvalid is an illegal operation type used for corruption detection. + OpInvalid uint8 = 0 + // OpPut represents a key-value put operation. + OpPut uint8 = 1 + // OpDelete represents a key deletion operation. + OpDelete uint8 = 2 +) + +// ValueKind represents how the value field is encoded in a WAL entry. +const ( + // VKNone indicates no value (used with Delete operations). + VKNone uint8 = 0 + // VKInline indicates the value field contains inline user bytes. + VKInline uint8 = 1 + // VKValueLogPointer indicates the value field contains an encoded Value Log pointer. + VKValueLogPointer uint8 = 2 +) diff --git a/wal/constants_test.go b/wal/constants_test.go new file mode 100644 index 0000000..ab15b18 --- /dev/null +++ b/wal/constants_test.go @@ -0,0 +1,72 @@ +package wal + +import "testing" + +func TestConstantValues(t *testing.T) { + tests := []struct { + name string + got interface{} + expected interface{} + }{ + {"WalBlockSize", WalBlockSize, 32 * 1024}, + {"WalFileHeaderSize", WalFileHeaderSize, 32}, + {"WalBatchHeaderSize", WalBatchHeaderSize, 18}, + {"PhysicalRecordHeaderSize", PhysicalRecordHeaderSize, 7}, + {"MaxWalBatchEntriesSize", MaxWalBatchEntriesSize, uint32(4 * 1024 * 1024)}, + {"MaxWalBatchEntryCount", MaxWalBatchEntryCount, uint32(10000)}, + {"MaxWalKeyBytes", MaxWalKeyBytes, uint32(4 * 1024)}, + {"MaxWalInlineValueBytes", MaxWalInlineValueBytes, uint32(4 * 1024)}, + {"MaxWalVarintBytes", MaxWalVarintBytes, 5}, + {"DefaultMaxWalSegmentSize", DefaultMaxWalSegmentSize, uint64(64 * 1024 * 1024)}, + {"WalMagic", WalMagic, uint32(0x57414C4B)}, + {"WalFormatVersion", WalFormatVersion, uint16(1)}, + } + + for _, tt := range tests { + if tt.got != tt.expected { + t.Errorf("%s = %v, want %v", tt.name, tt.got, tt.expected) + } + } +} + +func TestFragmentTypes(t *testing.T) { + if RecInvalid != uint8(0) { + t.Errorf("RecInvalid = %d, want 0", RecInvalid) + } + if RecFull != uint8(1) { + t.Errorf("RecFull = %d, want 1", RecFull) + } + if RecFirst != uint8(2) { + t.Errorf("RecFirst = %d, want 2", RecFirst) + } + if RecMiddle != uint8(3) { + t.Errorf("RecMiddle = %d, want 3", RecMiddle) + } + if RecLast != uint8(4) { + t.Errorf("RecLast = %d, want 4", RecLast) + } +} + +func TestOpTypes(t *testing.T) { + if OpInvalid != uint8(0) { + t.Errorf("OpInvalid = %d, want 0", OpInvalid) + } + if OpPut != uint8(1) { + t.Errorf("OpPut = %d, want 1", OpPut) + } + if OpDelete != uint8(2) { + t.Errorf("OpDelete = %d, want 2", OpDelete) + } +} + +func TestValueKinds(t *testing.T) { + if VKNone != uint8(0) { + t.Errorf("VKNone = %d, want 0", VKNone) + } + if VKInline != uint8(1) { + t.Errorf("VKInline = %d, want 1", VKInline) + } + if VKValueLogPointer != uint8(2) { + t.Errorf("VKValueLogPointer = %d, want 2", VKValueLogPointer) + } +} diff --git a/wal/doc.go b/wal/doc.go new file mode 100644 index 0000000..467f7ce --- /dev/null +++ b/wal/doc.go @@ -0,0 +1,2 @@ +// Package wal implements the Write-Ahead Log subsystem. +package wal diff --git a/wal/entry.go b/wal/entry.go new file mode 100644 index 0000000..bdc6cb0 --- /dev/null +++ b/wal/entry.go @@ -0,0 +1,134 @@ +package wal + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" +) + +// WalEntry represents a single WAL record. +type WalEntry struct { + OpType uint8 + ValueKind uint8 + Key []byte + Value []byte +} + +// Validate checks that the entry fields are consistent with the design rules. +func (e *WalEntry) Validate() error { + keyLen := len(e.Key) + if keyLen == 0 || keyLen > int(MaxWalKeyBytes) { + return fmt.Errorf("wal: invalid key length %d", keyLen) + } + + valLen := len(e.Value) + + switch e.OpType { + case OpPut: + switch e.ValueKind { + case VKInline: + if valLen > int(MaxWalInlineValueBytes) { + return fmt.Errorf("wal: inline value length %d out of range [0, %d]", valLen, MaxWalInlineValueBytes) + } + case VKValueLogPointer: + if valLen == 0 { + return errors.New("wal: value log pointer requires non-empty value") + } + default: + return fmt.Errorf("wal: put requires valueKind Inline(1) or ValueLogPointer(2), got %d", e.ValueKind) + } + + case OpDelete: + if e.ValueKind != VKNone { + return fmt.Errorf("wal: delete requires valueKind None(0), got %d", e.ValueKind) + } + if valLen != 0 { + return fmt.Errorf("wal: delete requires empty value, got length %d", valLen) + } + + default: + return fmt.Errorf("wal: invalid opType %d", e.OpType) + } + + return nil +} + +// EncodeEntry serializes a WalEntry into a byte slice. +func EncodeEntry(e *WalEntry) ([]byte, error) { + if err := e.Validate(); err != nil { + return nil, err + } + + keyLen := uint64(len(e.Key)) + valLen := uint64(len(e.Value)) + + // Size: 1 (opType) + 1 (valueKind) + varint(keyLen) + varint(valLen) + key + value + size := 2 + MaxWalVarintBytes + MaxWalVarintBytes + len(e.Key) + len(e.Value) + buf := make([]byte, size) + + buf[0] = e.OpType + buf[1] = e.ValueKind + n := 2 + n += binary.PutUvarint(buf[n:], keyLen) + n += binary.PutUvarint(buf[n:], valLen) + n += copy(buf[n:], e.Key) + n += copy(buf[n:], e.Value) + + return buf[:n], nil +} + +// DecodeEntry deserializes a WalEntry from a byte slice. +// Returns the decoded entry and the number of bytes consumed. +func DecodeEntry(data []byte) (entry *WalEntry, consumed int, err error) { + if len(data) < 2 { + return nil, 0, errors.New("wal: data too short for entry header") + } + + opType := data[0] + valueKind := data[1] + r := bytes.NewReader(data[2:]) + + keyLen, err := binary.ReadUvarint(r) + if err != nil { + return nil, 0, fmt.Errorf("wal: reading key length: %w", err) + } + valLen, err := binary.ReadUvarint(r) + if err != nil { + return nil, 0, fmt.Errorf("wal: reading value length: %w", err) + } + + // Calculate consumed so far: 2 header bytes + bytes read from reader + consumed = 2 + (len(data) - 2 - r.Len()) + + // Read key + remaining := len(data) - consumed + if uint64(remaining) < keyLen { + return nil, 0, fmt.Errorf("wal: data truncated: need %d bytes for key, have %d", keyLen, remaining) + } + key := make([]byte, keyLen) + copy(key, data[consumed:consumed+int(keyLen)]) + consumed += int(keyLen) + + // Read value + remaining = len(data) - consumed + if uint64(remaining) < valLen { + return nil, 0, fmt.Errorf("wal: data truncated: need %d bytes for value, have %d", valLen, remaining) + } + value := make([]byte, valLen) + copy(value, data[consumed:consumed+int(valLen)]) + consumed += int(valLen) + + e := &WalEntry{ + OpType: opType, + ValueKind: valueKind, + Key: key, + Value: value, + } + + if err := e.Validate(); err != nil { + return nil, 0, err + } + + return e, consumed, nil +} diff --git a/wal/entry_test.go b/wal/entry_test.go new file mode 100644 index 0000000..197d92b --- /dev/null +++ b/wal/entry_test.go @@ -0,0 +1,114 @@ +package wal + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestEntryRoundtrip(t *testing.T) { + maxKey := bytes.Repeat([]byte("k"), int(MaxWalKeyBytes)) + + cases := []struct { + name string + e *WalEntry + }{ + {"put_inline", &WalEntry{OpPut, VKInline, []byte("key1"), []byte("val1")}}, + {"put_inline_empty_value", &WalEntry{OpPut, VKInline, []byte("key2"), []byte{}}}, + {"delete", &WalEntry{OpDelete, VKNone, []byte("key3"), nil}}, + {"put_max_key", &WalEntry{OpPut, VKInline, maxKey, []byte("v")}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + encoded, err := EncodeEntry(tc.e) + if err != nil { + t.Fatalf("encode: %v", err) + } + + got, consumed, err := DecodeEntry(encoded) + if err != nil { + t.Fatalf("decode: %v", err) + } + if consumed != len(encoded) { + t.Fatalf("consumed %d != encoded len %d", consumed, len(encoded)) + } + + if got.OpType != tc.e.OpType { + t.Errorf("OpType: got %d, want %d", got.OpType, tc.e.OpType) + } + if got.ValueKind != tc.e.ValueKind { + t.Errorf("ValueKind: got %d, want %d", got.ValueKind, tc.e.ValueKind) + } + if !bytes.Equal(got.Key, tc.e.Key) { + t.Errorf("Key: got %q, want %q", got.Key, tc.e.Key) + } + if !bytes.Equal(got.Value, tc.e.Value) { + t.Errorf("Value: got %q, want %q", got.Value, tc.e.Value) + } + }) + } +} + +func TestEntryValidation(t *testing.T) { + bigKey := bytes.Repeat([]byte("k"), int(MaxWalKeyBytes)+1) + bigVal := bytes.Repeat([]byte("v"), int(MaxWalInlineValueBytes)+1) + + cases := []struct { + name string + e *WalEntry + wantErr bool + }{ + {"op_invalid", &WalEntry{OpInvalid, VKNone, []byte("k"), nil}, true}, + {"put_vk_none", &WalEntry{OpPut, VKNone, []byte("k"), nil}, true}, + {"delete_vk_inline", &WalEntry{OpDelete, VKInline, []byte("k"), nil}, true}, + {"key_empty", &WalEntry{OpPut, VKInline, []byte{}, []byte("v")}, true}, + {"key_too_big", &WalEntry{OpPut, VKInline, bigKey, []byte("v")}, true}, + {"put_inline_val_too_big", &WalEntry{OpPut, VKInline, []byte("k"), bigVal}, true}, + {"put_vlptr_val_empty", &WalEntry{OpPut, VKValueLogPointer, []byte("k"), []byte{}}, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := EncodeEntry(tc.e) + if (err != nil) != tc.wantErr { + t.Errorf("EncodeEntry() error = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} + +func TestEntryDecodeTruncated(t *testing.T) { + // Build a valid encoded entry, then truncate mid-varint. + e := &WalEntry{OpPut, VKInline, []byte("key1"), []byte("value1")} + full, err := EncodeEntry(e) + if err != nil { + t.Fatal(err) + } + + // Truncate to just 1 byte — not enough for header. + _, _, err = DecodeEntry(full[:1]) + if err == nil { + t.Error("expected error for 1-byte data") + } + + // Build data with an incomplete varint: opType + valueKind + start of varint (0xFF means more bytes follow). + trunc := []byte{OpPut, VKInline, 0xFF} + _, _, err = DecodeEntry(trunc) + if err == nil { + t.Error("expected error for truncated varint") + } + + // Also test: varint specifies more bytes than available. + // Encode a large key length varint but don't provide the key bytes. + varintBuf := make([]byte, binary.MaxVarintLen64) + n := binary.PutUvarint(varintBuf, 1000) // keyLen = 1000 + data := []byte{OpPut, VKInline} + data = append(data, varintBuf[:n]...) + data = append(data, varintBuf[:n]...) // valLen varint (also 1000) + // Don't append any key/value bytes. + _, _, err = DecodeEntry(data) + if err == nil { + t.Error("expected error for missing key/value bytes") + } +} diff --git a/wal/header.go b/wal/header.go new file mode 100644 index 0000000..b511387 --- /dev/null +++ b/wal/header.go @@ -0,0 +1,98 @@ +package wal + +import ( + "encoding/binary" + "errors" + "hash/crc32" +) + +const ( + walFileHeaderSize = 32 + walMagic = 0x57414C4B // "WALK" in ASCII + walFormatVersion = 1 +) + +// WalFileHeader is the 32-byte header written at the start of every WAL segment file. +type WalFileHeader struct { + Magic uint32 + FormatVersion uint16 + HeaderSize uint16 + BlockSize uint32 + SegmentID uint64 + StartSequence uint64 + HeaderCRC uint32 +} + +// EncodeWalHeader serializes h into a fixed-size 32-byte array using little-endian byte order. +// The HeaderCRC field is computed over bytes 0–27 (everything except the CRC itself). +func EncodeWalHeader(h *WalFileHeader) [walFileHeaderSize]byte { + h.HeaderSize = walFileHeaderSize + h.Magic = walMagic + h.FormatVersion = walFormatVersion + + var buf [walFileHeaderSize]byte + le := binary.LittleEndian + + le.PutUint32(buf[0:4], h.Magic) + le.PutUint16(buf[4:6], h.FormatVersion) + le.PutUint16(buf[6:8], h.HeaderSize) + le.PutUint32(buf[8:12], h.BlockSize) + le.PutUint64(buf[12:20], h.SegmentID) + le.PutUint64(buf[20:28], h.StartSequence) + + // CRC32 IEEE over bytes 0–27 (excludes the CRC field itself) + h.HeaderCRC = crc32.ChecksumIEEE(buf[0:28]) + le.PutUint32(buf[28:32], h.HeaderCRC) + + return buf +} + +var ( + errBadMagic = errors.New("wal: bad magic number") + errBadVersion = errors.New("wal: unsupported format version") + errBadHeaderSize = errors.New("wal: bad header size") + errCRCMismatch = errors.New("wal: header CRC mismatch") + errHeaderTooShort = errors.New("wal: header data too short") +) + +// DecodeWalHeader parses a 32-byte little-endian header and validates magic, +// format version, header size, and CRC. +func DecodeWalHeader(data []byte) (*WalFileHeader, error) { + if len(data) < walFileHeaderSize { + return nil, errHeaderTooShort + } + + le := binary.LittleEndian + + magic := le.Uint32(data[0:4]) + if magic != walMagic { + return nil, errBadMagic + } + + version := le.Uint16(data[4:6]) + if version != walFormatVersion { + return nil, errBadVersion + } + + hdrSize := le.Uint16(data[6:8]) + if hdrSize != walFileHeaderSize { + return nil, errBadHeaderSize + } + + // Verify CRC before trusting any other fields + gotCRC := crc32.ChecksumIEEE(data[0:28]) + storedCRC := le.Uint32(data[28:32]) + if gotCRC != storedCRC { + return nil, errCRCMismatch + } + + return &WalFileHeader{ + Magic: magic, + FormatVersion: version, + HeaderSize: hdrSize, + BlockSize: le.Uint32(data[8:12]), + SegmentID: le.Uint64(data[12:20]), + StartSequence: le.Uint64(data[20:28]), + HeaderCRC: storedCRC, + }, nil +} diff --git a/wal/header_test.go b/wal/header_test.go new file mode 100644 index 0000000..5fd03d8 --- /dev/null +++ b/wal/header_test.go @@ -0,0 +1,87 @@ +package wal + +import ( + "errors" + "testing" +) + +func TestHeaderRoundtrip(t *testing.T) { + orig := &WalFileHeader{ + BlockSize: 32 * 1024, // 32 KB + SegmentID: 5, + StartSequence: 1000, + } + + encoded := EncodeWalHeader(orig) + decoded, err := DecodeWalHeader(encoded[:]) + if err != nil { + t.Fatalf("DecodeWalHeader returned error: %v", err) + } + + if decoded.Magic != walMagic { + t.Errorf("Magic = %x, want %x", decoded.Magic, walMagic) + } + if decoded.FormatVersion != walFormatVersion { + t.Errorf("FormatVersion = %d, want %d", decoded.FormatVersion, walFormatVersion) + } + if decoded.HeaderSize != walFileHeaderSize { + t.Errorf("HeaderSize = %d, want %d", decoded.HeaderSize, walFileHeaderSize) + } + if decoded.BlockSize != orig.BlockSize { + t.Errorf("BlockSize = %d, want %d", decoded.BlockSize, orig.BlockSize) + } + if decoded.SegmentID != orig.SegmentID { + t.Errorf("SegmentID = %d, want %d", decoded.SegmentID, orig.SegmentID) + } + if decoded.StartSequence != orig.StartSequence { + t.Errorf("StartSequence = %d, want %d", decoded.StartSequence, orig.StartSequence) + } + if decoded.HeaderCRC != orig.HeaderCRC { + t.Errorf("HeaderCRC = %x, want %x", decoded.HeaderCRC, orig.HeaderCRC) + } +} + +func TestHeaderCRC(t *testing.T) { + h := &WalFileHeader{ + BlockSize: 32 * 1024, + SegmentID: 1, + StartSequence: 0, + } + encoded := EncodeWalHeader(h) + + // Flip a byte in the magic field (bytes 0-3) + encoded[0] ^= 0xFF + + _, err := DecodeWalHeader(encoded[:]) + if !errors.Is(err, errCRCMismatch) && !errors.Is(err, errBadMagic) { + // Flipping magic may fail on magic check first or CRC check + // Either way, decoding must fail + t.Fatalf("expected CRC or magic error, got: %v", err) + } + + // Restore magic and flip a byte in the payload instead + encoded[0] = byte(walMagic & 0xFF) + encoded[12] ^= 0x01 // flip byte in SegmentID + + _, err = DecodeWalHeader(encoded[:]) + if !errors.Is(err, errCRCMismatch) { + t.Fatalf("expected errCRCMismatch, got: %v", err) + } +} + +func TestHeaderBadMagic(t *testing.T) { + data := make([]byte, 32) + // All zeros — magic won't match + _, err := DecodeWalHeader(data) + if !errors.Is(err, errBadMagic) { + t.Fatalf("expected errBadMagic, got: %v", err) + } +} + +func TestHeaderShortData(t *testing.T) { + data := make([]byte, 16) // too short + _, err := DecodeWalHeader(data) + if !errors.Is(err, errHeaderTooShort) { + t.Fatalf("expected errHeaderTooShort, got: %v", err) + } +} diff --git a/wal/record.go b/wal/record.go new file mode 100644 index 0000000..c4bf994 --- /dev/null +++ b/wal/record.go @@ -0,0 +1,126 @@ +package wal + +import ( + "encoding/binary" + "errors" + "hash/crc32" +) + +// PhysicalRecord represents a single physical record in the WAL. +type PhysicalRecord struct { + CRC uint32 + Length uint16 + Type uint8 + Payload []byte +} + +// EncodePhysicalRecord encodes a physical record with the given type and payload. +// Format: [crc32 u32 LE][length u16 LE][type u8][payload bytes] +// CRC covers length + type + payload. +func EncodePhysicalRecord(recType uint8, payload []byte) []byte { + length := uint16(len(payload)) + buf := make([]byte, PhysicalRecordHeaderSize+len(payload)) + + // Write length and type first so we can compute CRC. + binary.LittleEndian.PutUint16(buf[4:6], length) + buf[6] = recType + copy(buf[7:], payload) + + // CRC covers bytes [4:] = length + type + payload. + crc := crc32.ChecksumIEEE(buf[4:]) + binary.LittleEndian.PutUint32(buf[0:4], crc) + + return buf +} + +// DecodePhysicalRecord decodes a physical record from data. +// Returns the record, number of bytes consumed, and any error. +func DecodePhysicalRecord(data []byte) (rec *PhysicalRecord, consumed int, err error) { + if len(data) < PhysicalRecordHeaderSize { + return nil, 0, errors.New("record: data too short for header") + } + + crc := binary.LittleEndian.Uint32(data[0:4]) + length := binary.LittleEndian.Uint16(data[4:6]) + recType := data[6] + + if int(length) > len(data)-PhysicalRecordHeaderSize { + return nil, 0, errors.New("record: data too short for payload") + } + + payload := make([]byte, length) + copy(payload, data[7:7+length]) + + // Verify CRC: covers length + type + payload. + expectedCRC := crc32.ChecksumIEEE(data[4 : 7+length]) + if crc != expectedCRC { + return nil, 0, errors.New("record: CRC mismatch") + } + + consumed = PhysicalRecordHeaderSize + int(length) + return &PhysicalRecord{ + CRC: crc, + Length: length, + Type: recType, + Payload: payload, + }, consumed, nil +} + +// PaddingNeeded returns the number of padding bytes needed at blockOffset. +// If the remaining space in the current block is <= PhysicalRecordHeaderSize (7), +// that remaining space must be zero-padded. +func PaddingNeeded(blockOffset uint32) int { + remaining := WalBlockSize - (blockOffset % WalBlockSize) + if remaining <= PhysicalRecordHeaderSize { + return int(remaining) + } + return 0 +} + +// CanFitRecord reports whether a physical record with the given payload length +// can fit in the current block starting at blockOffset. +func CanFitRecord(blockOffset uint32, payloadLen uint32) bool { + remaining := WalBlockSize - (blockOffset % WalBlockSize) + return int(remaining) >= PhysicalRecordHeaderSize+int(payloadLen) +} + +// SplitIntoRecords splits an encoded WAL batch into physical record payloads +// respecting 32 KB block boundaries. +// Each returned byte slice is the full encoded physical record (header + payload). +func SplitIntoRecords(encodedBatch []byte) [][]byte { + maxPayload := WalBlockSize - PhysicalRecordHeaderSize + total := len(encodedBatch) + + if total == 0 { + return nil + } + + // Single record fits entirely. + if total <= maxPayload { + return [][]byte{EncodePhysicalRecord(RecFull, encodedBatch)} + } + + var records [][]byte + offset := 0 + + for offset < total { + chunkLen := min(total-offset, maxPayload) + + var recType uint8 + switch { + case offset == 0 && offset+chunkLen == total: + recType = RecFull + case offset == 0: + recType = RecFirst + case offset+chunkLen == total: + recType = RecLast + default: + recType = RecMiddle + } + + records = append(records, EncodePhysicalRecord(recType, encodedBatch[offset:offset+chunkLen])) + offset += chunkLen + } + + return records +} diff --git a/wal/record_test.go b/wal/record_test.go new file mode 100644 index 0000000..aebc362 --- /dev/null +++ b/wal/record_test.go @@ -0,0 +1,143 @@ +package wal + +import ( + "bytes" + "testing" +) + +func TestRecordRoundtrip(t *testing.T) { + payload := []byte("hello world") + encoded := EncodePhysicalRecord(RecFull, payload) + + rec, consumed, err := DecodePhysicalRecord(encoded) + if err != nil { + t.Fatalf("DecodePhysicalRecord failed: %v", err) + } + + if rec.Type != RecFull { + t.Errorf("expected type RecFull(%d), got %d", RecFull, rec.Type) + } + if string(rec.Payload) != "hello world" { + t.Errorf("expected payload 'hello world', got %q", string(rec.Payload)) + } + if consumed != 7+len(payload) { + t.Errorf("expected consumed %d, got %d", 7+len(payload), consumed) + } +} + +func TestSplitSmallPayload(t *testing.T) { + payload := make([]byte, 100) + for i := range payload { + payload[i] = byte(i) + } + + records := SplitIntoRecords(payload) + if len(records) != 1 { + t.Fatalf("expected 1 record, got %d", len(records)) + } + + rec, _, err := DecodePhysicalRecord(records[0]) + if err != nil { + t.Fatalf("DecodePhysicalRecord failed: %v", err) + } + if rec.Type != RecFull { + t.Errorf("expected RecFull, got %d", rec.Type) + } + if !bytes.Equal(rec.Payload, payload) { + t.Error("payload mismatch") + } +} + +func TestSplitIntoRecords(t *testing.T) { + // 40 KB payload → needs to split across blocks. + payload := make([]byte, 40*1024) + for i := range payload { + payload[i] = byte(i % 256) + } + + records := SplitIntoRecords(payload) + if len(records) < 2 { + t.Fatalf("expected at least 2 records, got %d", len(records)) + } + + // Verify fragment sequence. + types := make([]uint8, len(records)) + var concatenated []byte + for i, enc := range records { + rec, _, err := DecodePhysicalRecord(enc) + if err != nil { + t.Fatalf("DecodePhysicalRecord record %d failed: %v", i, err) + } + types[i] = rec.Type + concatenated = append(concatenated, rec.Payload...) + } + + // First record must be RecFirst. + if types[0] != RecFirst { + t.Errorf("first record type: expected RecFirst(%d), got %d", RecFirst, types[0]) + } + // Last record must be RecLast. + if types[len(types)-1] != RecLast { + t.Errorf("last record type: expected RecLast(%d), got %d", RecLast, types[len(types)-1]) + } + // Middle records must be RecMiddle. + for i := 1; i < len(types)-1; i++ { + if types[i] != RecMiddle { + t.Errorf("record %d type: expected RecMiddle(%d), got %d", i, RecMiddle, types[i]) + } + } + + // Concatenated payloads must equal original. + if !bytes.Equal(concatenated, payload) { + t.Error("concatenated payloads do not match original") + } +} + +func TestBlockPadding(t *testing.T) { + // blockOffset = WalBlockSize - 5 → remaining = 5, which is <= 7 → padding needed = 5. + blockOffset := uint32(WalBlockSize - 5) + padding := PaddingNeeded(blockOffset) + if padding != 5 { + t.Errorf("PaddingNeeded(%d): expected 5, got %d", blockOffset, padding) + } + + // Cannot fit a record. + if CanFitRecord(blockOffset, 1) { + t.Error("CanFitRecord should return false when remaining <= 7") + } + + // blockOffset = WalBlockSize - 8 → remaining = 8, which is > 7 → no padding needed. + blockOffset2 := uint32(WalBlockSize - 8) + padding2 := PaddingNeeded(blockOffset2) + if padding2 != 0 { + t.Errorf("PaddingNeeded(%d): expected 0, got %d", blockOffset2, padding2) + } + + // Can fit a 1-byte payload: remaining=8, header=7, payload=1 → 8 >= 8. + if !CanFitRecord(blockOffset2, 1) { + t.Error("CanFitRecord should return true when remaining=8 and payloadLen=1") + } + + // Cannot fit a 2-byte payload: remaining=8, header=7, payload=2 → 8 < 9. + if CanFitRecord(blockOffset2, 2) { + t.Error("CanFitRecord should return false when remaining=8 and payloadLen=2") + } +} + +func TestCRCMismatch(t *testing.T) { + encoded := EncodePhysicalRecord(RecFull, []byte("test")) + // Corrupt a payload byte. + encoded[8] ^= 0xFF + + _, _, err := DecodePhysicalRecord(encoded) + if err == nil { + t.Error("expected CRC mismatch error") + } +} + +func TestDataTooShort(t *testing.T) { + _, _, err := DecodePhysicalRecord([]byte{1, 2, 3}) + if err == nil { + t.Error("expected error for data too short") + } +} diff --git a/wal/sequence.go b/wal/sequence.go new file mode 100644 index 0000000..c2317e9 --- /dev/null +++ b/wal/sequence.go @@ -0,0 +1,87 @@ +package wal + +import ( + "sync/atomic" + + "github.com/dailz/go-kv" +) + +// SequenceManager manages monotonic sequence number allocation for the WAL. +// Invariant: durableSequence <= publishedSequence <= nextSequence +type SequenceManager struct { + nextSequence atomic.Uint64 + publishedSequence atomic.Uint64 + durableSequence atomic.Uint64 + exhausted atomic.Bool +} + +// NewSequenceManager creates a SequenceManager initialised from a recovered +// sequence number. All three watermarks start at recoveredSequence. +func NewSequenceManager(recoveredSequence uint64) *SequenceManager { + sm := &SequenceManager{} + sm.nextSequence.Store(recoveredSequence) + sm.publishedSequence.Store(recoveredSequence) + sm.durableSequence.Store(recoveredSequence) + return sm +} + +// AllocateBatch atomically reserves [base, base+count-1] sequence numbers. +// Returns ErrSequenceExhausted if count is 0 or the allocation would overflow uint64. +func (sm *SequenceManager) AllocateBatch(count uint32) (baseSequence uint64, err error) { + if count == 0 { + return 0, go_kv.ErrSequenceExhausted + } + for { + if sm.exhausted.Load() { + return 0, go_kv.ErrSequenceExhausted + } + base := sm.nextSequence.Load() + last := base + uint64(count) - 1 + if last < base { + return 0, go_kv.ErrSequenceExhausted + } + newNext := last + 1 + if !sm.nextSequence.CompareAndSwap(base, newNext) { + continue + } + if newNext == 0 { + sm.exhausted.Store(true) + } + return base, nil + } +} + +// Publish advances the publishedSequence watermark to seq (only forward). +func (sm *SequenceManager) Publish(seq uint64) { + for { + current := sm.publishedSequence.Load() + if seq <= current { + return + } + if sm.publishedSequence.CompareAndSwap(current, seq) { + return + } + } +} + +// MarkDurable advances the durableSequence watermark to seq (only forward). +func (sm *SequenceManager) MarkDurable(seq uint64) { + for { + current := sm.durableSequence.Load() + if seq <= current { + return + } + if sm.durableSequence.CompareAndSwap(current, seq) { + return + } + } +} + +// Published returns the current publishedSequence watermark. +func (sm *SequenceManager) Published() uint64 { return sm.publishedSequence.Load() } + +// Durable returns the current durableSequence watermark. +func (sm *SequenceManager) Durable() uint64 { return sm.durableSequence.Load() } + +// NextSequence returns the next sequence number to be allocated. +func (sm *SequenceManager) NextSequence() uint64 { return sm.nextSequence.Load() } diff --git a/wal/sequence_test.go b/wal/sequence_test.go new file mode 100644 index 0000000..36c0381 --- /dev/null +++ b/wal/sequence_test.go @@ -0,0 +1,154 @@ +package wal + +import ( + "errors" + "math" + "sync" + "sync/atomic" + "testing" + + "github.com/dailz/go-kv" +) + +func TestSequenceAllocation(t *testing.T) { + sm := NewSequenceManager(0) + + base, err := sm.AllocateBatch(5) + if err != nil { + t.Fatalf("AllocateBatch(5): %v", err) + } + if base != 0 { + t.Fatalf("expected base=0, got %d", base) + } + + base, err = sm.AllocateBatch(3) + if err != nil { + t.Fatalf("AllocateBatch(3): %v", err) + } + if base != 5 { + t.Fatalf("expected base=5, got %d", base) + } + + base, err = sm.AllocateBatch(1) + if err != nil { + t.Fatalf("AllocateBatch(1): %v", err) + } + if base != 8 { + t.Fatalf("expected base=8, got %d", base) + } + + if sm.NextSequence() != 9 { + t.Fatalf("expected NextSequence=9, got %d", sm.NextSequence()) + } +} + +func TestSequenceOverflow(t *testing.T) { + nearMax := uint64(math.MaxUint64 - 2) + sm := NewSequenceManager(nearMax) + + // Remaining: MaxUint64-2, MaxUint64-1, MaxUint64 = 3 slots. + // Asking for 5 should overflow. + _, err := sm.AllocateBatch(5) + if !errors.Is(err, go_kv.ErrSequenceExhausted) { + t.Fatalf("expected ErrSequenceExhausted, got %v", err) + } + + // 3 should still succeed. + base, err := sm.AllocateBatch(3) + if err != nil { + t.Fatalf("AllocateBatch(3): %v", err) + } + if base != nearMax { + t.Fatalf("expected base=%d, got %d", nearMax, base) + } + + // Now any further allocation should fail. + _, err = sm.AllocateBatch(1) + if !errors.Is(err, go_kv.ErrSequenceExhausted) { + t.Fatalf("expected ErrSequenceExhausted after exhaustion, got %v", err) + } +} + +func TestPublishAdvance(t *testing.T) { + sm := NewSequenceManager(0) + + sm.Publish(10) + if sm.Published() != 10 { + t.Fatalf("expected Published=10, got %d", sm.Published()) + } + + // Publishing a lower value must not decrease the watermark. + sm.Publish(5) + if sm.Published() != 10 { + t.Fatalf("expected Published=10 (no decrease), got %d", sm.Published()) + } + + sm.Publish(15) + if sm.Published() != 15 { + t.Fatalf("expected Published=15, got %d", sm.Published()) + } +} + +func TestMarkDurable(t *testing.T) { + sm := NewSequenceManager(0) + + sm.MarkDurable(8) + if sm.Durable() != 8 { + t.Fatalf("expected Durable=8, got %d", sm.Durable()) + } + + // Lower value must not decrease. + sm.MarkDurable(3) + if sm.Durable() != 8 { + t.Fatalf("expected Durable=8 (no decrease), got %d", sm.Durable()) + } +} + +func TestZeroCountRejected(t *testing.T) { + sm := NewSequenceManager(0) + + _, err := sm.AllocateBatch(0) + if !errors.Is(err, go_kv.ErrSequenceExhausted) { + t.Fatalf("expected ErrSequenceExhausted for count=0, got %v", err) + } +} + +func TestConcurrentAllocation(t *testing.T) { + const goroutines = 16 + const batchSize uint32 = 100 + + sm := NewSequenceManager(0) + + var totalAllocated atomic.Uint64 + var wg sync.WaitGroup + wg.Add(goroutines) + + for i := 0; i < goroutines; i++ { + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + base, err := sm.AllocateBatch(batchSize) + if err != nil { + t.Errorf("AllocateBatch failed: %v", err) + return + } + totalAllocated.Add(uint64(batchSize)) + + // Verify no overlap: base must be aligned to batchSize increments + // and within valid range. The key property is no gaps. + _ = base + } + }() + } + + wg.Wait() + + expected := uint64(goroutines) * 50 * uint64(batchSize) + if totalAllocated.Load() != expected { + t.Fatalf("expected total allocated=%d, got %d", expected, totalAllocated.Load()) + } + + if sm.NextSequence() != expected { + t.Fatalf("expected NextSequence=%d, got %d", expected, sm.NextSequence()) + } +}