fix: implement group commit collection window (500µs or 32KB)
Replace polling loop (Collect + Sleep) with proper batch collection: - Block on first request via channel receive - Start 500µs timer for collection window - Accumulate requests until timer fires or batch reaches 32KB - Shutdown handling at all blocking points Design doc §3.2.5: '等待组提交触发(500µs 或 32KB,先到者触发)' Add GroupCommitDelay config field (default 500µs, must be > 0 and < 10ms).
This commit is contained in:
+52
-9
@@ -6,8 +6,8 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dailz/go-kv/errkit"
|
||||
"github.com/dailz/go-kv/config"
|
||||
"github.com/dailz/go-kv/errkit"
|
||||
"github.com/dailz/go-kv/memtable"
|
||||
)
|
||||
|
||||
@@ -101,24 +101,67 @@ func NewWalWriter(cfg *config.WalConfig, dir string, startSegmentID uint64, star
|
||||
|
||||
func (ww *WalWriter) runLoop() {
|
||||
defer ww.wg.Done()
|
||||
const batchSizeThreshold = 32 * 1024
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ww.done:
|
||||
ww.processRemaining()
|
||||
return
|
||||
default:
|
||||
}
|
||||
case req, ok := <-ww.queue.ch:
|
||||
if !ok {
|
||||
ww.processRemaining()
|
||||
return
|
||||
}
|
||||
|
||||
requests := ww.queue.Collect()
|
||||
if len(requests) == 0 {
|
||||
time.Sleep(100 * time.Microsecond)
|
||||
continue
|
||||
}
|
||||
requests := []*CommitRequest{req}
|
||||
accumulatedSize := estimateRequestSize(req)
|
||||
timer := time.NewTimer(ww.cfg.GroupCommitDelay)
|
||||
|
||||
ww.processBatch(requests)
|
||||
collect:
|
||||
for accumulatedSize < batchSizeThreshold {
|
||||
select {
|
||||
case <-timer.C:
|
||||
break collect
|
||||
case req, ok := <-ww.queue.ch:
|
||||
if !ok {
|
||||
break collect
|
||||
}
|
||||
requests = append(requests, req)
|
||||
accumulatedSize += estimateRequestSize(req)
|
||||
case <-ww.done:
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
ww.sendError(requests, errkit.ErrWriteStopped)
|
||||
ww.processRemaining()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
ww.processBatch(requests)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func estimateRequestSize(req *CommitRequest) int {
|
||||
size := 0
|
||||
for _, entry := range req.Entries {
|
||||
size += len(entry.Key) + len(entry.Value) + 4
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (ww *WalWriter) processRemaining() {
|
||||
for {
|
||||
requests := ww.queue.Collect()
|
||||
|
||||
Reference in New Issue
Block a user