b612/notify
1
0
Fork 0
Code Issues Pull Requests Projects Releases 11 Wiki Activity

feat: 完善 RecordStream 的协议协商、运行观测与文档说明

Browse Source 
- 将 RecordStream 出站路径收敛为单 writer loop
  - 支持在 batch header 中 piggyback AckSeq，保留独立 ack 作为兼容回退
  - 增加 record stream 打开阶段能力协商，支持 mixed-version peer 自动降级
  - 补充 RecordSnapshot 与 diagnostics summary 的 record-plane 观测项
  - 增加 batch/ack/error frame、piggyback ack、barrier 等待拆分与 apply backlog 指标
  - 收紧 TransportConn detach 后的 runtime snapshot 语义
  - 补充 README 中的 RecordStream 语义、兼容行为与诊断快照说明
  - 补充相关单测与 race 回归验证
This commit is contained in:
兔子
2026-04-15 19:52:45 +08:00
parent 09d972c7b7
commit 7ed3dd5b37
16 changed files with 1341 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
record_stream.go
+231 -109
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"sync"
"sync/atomic"
"time"
)
@@ -103,25 +104,47 @@ type recordConfig struct {
type recordFlushRequest struct {
targetSeq uint64
forceAck bool
done chan error
}
type recordObservability struct {
batchFramesSent atomic.Int64
ackFramesSent atomic.Int64
errorFramesSent atomic.Int64
batchFramesReceived atomic.Int64
ackFramesReceived atomic.Int64
errorFramesReceived atomic.Int64
piggybackAckSent atomic.Int64
piggybackAckReceived atomic.Int64
barrierCount atomic.Int64
barrierFlushWaitNanos atomic.Int64
barrierApplyWaitNanos atomic.Int64
}
type recordStream struct {
stream Stream
ctx context.Context
cancel context.CancelFunc
cfg recordConfig
writeMu sync.Mutex
sendCh chan recordOutboundMessage
flushCh chan recordFlushRequest
recvCh chan RecordMessage
ackCh chan struct{}
readerCh chan struct{}
stream Stream
ctx context.Context
cancel context.CancelFunc
cfg recordConfig
writeMu sync.Mutex
sendCh chan recordOutboundMessage
flushCh chan recordFlushRequest
recvCh chan RecordMessage
ackCh chan struct{}
readerCh chan struct{}
useBatchAck bool
obs recordObservability
mu sync.Mutex
stateNotify chan struct{}
runtime *recordRuntime
runtimeKey string
runtimeWatchOnce sync.Once
runtimeDetachOnce sync.Once
nextOutboundSeq uint64
enqueuedOutboundSeq uint64
flushedOutboundSeq uint64
@@ -135,6 +158,7 @@ type recordStream struct {
inboundAppliedSeq uint64
inboundApplied map[uint64]struct{}
inboundAckSentSeq uint64
maxPendingApply int
remoteClosed bool
readErr error
@@ -193,6 +217,7 @@ func recordConfigFromOptions(opt RecordOpenOptions) recordConfig {
func normalizeRecordStreamOpenOptions(opt StreamOpenOptions) StreamOpenOptions {
opt.Channel = StreamRecordChannel
opt.Metadata = advertiseRecordStreamOpenMetadata(opt.Metadata)
return opt
}
@@ -207,22 +232,22 @@ func WrapStreamAsRecord(stream Stream, opt RecordOpenOptions) (RecordStream, err
}
ctx, cancel := context.WithCancel(parent)
record := &recordStream{
stream: stream,
ctx: ctx,
cancel: cancel,
cfg: recordConfigFromOptions(opt),
sendCh: make(chan recordOutboundMessage, opt.MaxBatchRecords*2),
flushCh: make(chan recordFlushRequest),
recvCh: make(chan RecordMessage, opt.InboundQueueLimit),
ackCh: make(chan struct{}, 1),
readerCh: make(chan struct{}),
stream: stream,
ctx: ctx,
cancel: cancel,
cfg: recordConfigFromOptions(opt),
sendCh: make(chan recordOutboundMessage, opt.MaxBatchRecords*2),
flushCh: make(chan recordFlushRequest),
recvCh: make(chan RecordMessage, opt.InboundQueueLimit),
ackCh: make(chan struct{}, 1),
readerCh: make(chan struct{}),
useBatchAck: recordStreamUseBatchAck(stream.Metadata()),
stateNotify: make(chan struct{}),
outstandingSizes: make(map[uint64]int),
inboundApplied: make(map[uint64]struct{}),
}
go record.sendLoop()
go record.ackLoop()
go record.writerLoop()
go record.readLoop()
return record, nil
}
@@ -360,13 +385,20 @@ func (r *recordStream) BarrierTo(ctx context.Context, target uint64) (uint64, er
if target > current {
return 0, errRecordSeqInvalid
}
if err := r.Flush(ctx); err != nil {
r.obs.barrierCount.Add(1)
flushStart := time.Now()
err := r.Flush(ctx)
r.obs.barrierFlushWaitNanos.Add(time.Since(flushStart).Nanoseconds())
if err != nil {
return 0, err
}
if target == 0 {
return 0, nil
}
if err := r.waitAckedAtLeast(ctx, target); err != nil {
applyStart := time.Now()
err = r.waitAckedAtLeast(ctx, target)
r.obs.barrierApplyWaitNanos.Add(time.Since(applyStart).Nanoseconds())
if err != nil {
return 0, err
}
return target, nil
@@ -520,54 +552,118 @@ func (r *recordStream) waitAckedAtLeast(ctx context.Context, target uint64) erro
}
}
func (r *recordStream) sendLoop() {
func (r *recordStream) writerLoop() {
var (
batch []recordOutboundMessage
batches int
bytes int
timer *time.Timer
timerCh <-chan time.Time
batch []recordOutboundMessage
batches int
bytes int
batchTimer *time.Timer
batchTimerCh <-chan time.Time
ackTimer *time.Timer
ackTimerCh <-chan time.Time
)
stopTimer := func() {
if timer == nil {
stopBatchTimer := func() {
if batchTimer == nil {
return
}
if !timer.Stop() {
if !batchTimer.Stop() {
select {
case <-timer.C:
case <-batchTimer.C:
default:
}
}
timerCh = nil
batchTimerCh = nil
}
flush := func() error {
if len(batch) == 0 {
stopAckTimer := func() {
if ackTimer == nil {
return
}
if !ackTimer.Stop() {
select {
case <-ackTimer.C:
default:
}
}
ackTimerCh = nil
}
scheduleAck := func(hasPendingBatch bool, force bool) (uint64, bool) {
ackSeq := r.pendingAckSeq()
if ackSeq == 0 {
stopAckTimer()
return 0, false
}
if force {
stopAckTimer()
return ackSeq, true
}
if hasPendingBatch && r.useBatchAck {
stopAckTimer()
return 0, false
}
if r.shouldSendAckNow() || r.cfg.AckDelay <= 0 {
stopAckTimer()
return ackSeq, true
}
if ackTimer == nil {
ackTimer = time.NewTimer(r.cfg.AckDelay)
} else {
ackTimer.Reset(r.cfg.AckDelay)
}
ackTimerCh = ackTimer.C
return 0, false
}
sendStandaloneAck := func(ackSeq uint64) error {
if ackSeq == 0 {
return nil
}
payload, err := encodeRecordBatchFrame(batch)
payload, err := encodeRecordAckFrame(ackSeq)
if err != nil {
return err
}
if err := r.writePayloadFrame(payload); err != nil {
return err
}
r.obs.ackFramesSent.Add(1)
r.markAckSent(ackSeq)
return nil
}
flushBatch := func() error {
if len(batch) == 0 {
return nil
}
ackSeq := r.pendingAckSeq()
payload, err := encodeRecordBatchFrame(batch, ackSeq, r.useBatchAck)
if err != nil {
return err
}
if err := r.writePayloadFrame(payload); err != nil {
return err
}
r.obs.batchFramesSent.Add(1)
if r.useBatchAck && ackSeq != 0 {
r.obs.piggybackAckSent.Add(1)
r.markAckSent(ackSeq)
}
r.markFlushed(batch[len(batch)-1].Seq)
batch = nil
batches = 0
bytes = 0
stopTimer()
stopBatchTimer()
if ackSeq, sendNow := scheduleAck(false, false); sendNow {
return sendStandaloneAck(ackSeq)
}
return nil
}
flushUntil := func(target uint64) error {
for {
if target == 0 {
return flush()
return flushBatch()
}
if r.flushedAtLeast(target) {
return nil
}
if len(batch) > 0 && batch[len(batch)-1].Seq >= target {
if err := flush(); err != nil {
if err := flushBatch(); err != nil {
return err
}
if r.flushedAtLeast(target) {
@@ -583,7 +679,7 @@ func (r *recordStream) sendLoop() {
batches++
bytes += len(req.Payload)
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flush(); err != nil {
if err := flushBatch(); err != nil {
return err
}
}
@@ -598,72 +694,54 @@ func (r *recordStream) sendLoop() {
batches++
bytes += len(req.Payload)
if len(batch) == 1 && r.cfg.MaxBatchDelay > 0 {
if timer == nil {
timer = time.NewTimer(r.cfg.MaxBatchDelay)
if batchTimer == nil {
batchTimer = time.NewTimer(r.cfg.MaxBatchDelay)
} else {
timer.Reset(r.cfg.MaxBatchDelay)
batchTimer.Reset(r.cfg.MaxBatchDelay)
}
timerCh = timer.C
batchTimerCh = batchTimer.C
}
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flush(); err != nil {
r.setTerminalError(err)
return
}
}
case req := <-r.flushCh:
req.done <- flushUntil(req.targetSeq)
case <-timerCh:
if err := flush(); err != nil {
r.setTerminalError(err)
return
}
}
}
}
func (r *recordStream) ackLoop() {
var (
timer *time.Timer
timerCh <-chan time.Time
)
stopTimer := func() {
if timer == nil {
return
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timerCh = nil
}
for {
select {
case <-r.ctx.Done():
return
case <-r.ackCh:
if r.shouldSendAckNow() {
stopTimer()
if err := r.flushAckNow(); err != nil {
if err := flushBatch(); err != nil {
r.setTerminalError(err)
return
}
continue
}
if timer == nil {
timer = time.NewTimer(r.cfg.AckDelay)
} else {
timer.Reset(r.cfg.AckDelay)
if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
return
}
}
timerCh = timer.C
case <-timerCh:
stopTimer()
if err := r.flushAckNow(); err != nil {
case req := <-r.flushCh:
err := flushUntil(req.targetSeq)
if err == nil && req.forceAck {
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
err = sendStandaloneAck(ackSeq)
}
}
req.done <- err
case <-batchTimerCh:
if err := flushBatch(); err != nil {
r.setTerminalError(err)
return
}
case <-r.ackCh:
if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
return
}
}
case <-ackTimerCh:
stopAckTimer()
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
return
}
}
}
}
}
@@ -694,6 +772,15 @@ func (r *recordStream) readLoop() {
}
switch frame.Type {
case recordFrameTypeBatch:
r.obs.batchFramesReceived.Add(1)
if frame.AckSeq != 0 {
r.obs.piggybackAckReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err)
_ = r.stream.Reset(err)
return
}
}
if err := r.handleBatchFrame(frame.Batch); err != nil {
_ = r.sendFailureFrame(RecordFailure{
FailedSeq: r.nextInboundFailureSeq(),
@@ -705,12 +792,14 @@ func (r *recordStream) readLoop() {
return
}
case recordFrameTypeAck:
r.obs.ackFramesReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err)
_ = r.stream.Reset(err)
return
}
case recordFrameTypeError:
r.obs.errorFramesReceived.Add(1)
r.setReadError(frame.Failure)
return
default:
@@ -732,6 +821,7 @@ func (r *recordStream) handleBatchFrame(batch []recordOutboundMessage) error {
}
lastSeq := batch[len(batch)-1].Seq
r.inboundReceivedSeq = lastSeq
r.updatePendingApplyLocked()
r.signalStateLocked()
r.mu.Unlock()
for _, item := range batch {
@@ -889,23 +979,21 @@ func (r *recordStream) shouldSendAckNow() bool {
return r.inboundAppliedSeq > r.inboundAckSentSeq && int(r.inboundAppliedSeq-r.inboundAckSentSeq) >= r.cfg.AckEveryRecords
}
func (r *recordStream) flushAckNow() error {
func (r *recordStream) pendingAckSeq() uint64 {
if r == nil {
return errRecordStreamNil
return 0
}
r.mu.Lock()
ackSeq := r.inboundAppliedSeq
if ackSeq <= r.inboundAckSentSeq {
r.mu.Unlock()
return nil
defer r.mu.Unlock()
if r.inboundAppliedSeq <= r.inboundAckSentSeq {
return 0
}
r.mu.Unlock()
payload, err := encodeRecordAckFrame(ackSeq)
if err != nil {
return err
}
if err := r.writePayloadFrame(payload); err != nil {
return err
return r.inboundAppliedSeq
}
func (r *recordStream) markAckSent(ackSeq uint64) {
if r == nil || ackSeq == 0 {
return
}
r.mu.Lock()
if ackSeq > r.inboundAckSentSeq {
@@ -913,7 +1001,27 @@ func (r *recordStream) flushAckNow() error {
r.signalStateLocked()
}
r.mu.Unlock()
return nil
}
func (r *recordStream) flushAckNow() error {
if r == nil {
return errRecordStreamNil
}
req := recordFlushRequest{
forceAck: true,
done: make(chan error, 1),
}
select {
case <-r.ctx.Done():
return r.streamError()
case r.flushCh <- req:
}
select {
case <-r.ctx.Done():
return r.streamError()
case err := <-req.done:
return err
}
}
func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
@@ -921,7 +1029,11 @@ func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
if err != nil {
return err
}
return r.writePayloadFrame(payload)
if err := r.writePayloadFrame(payload); err != nil {
return err
}
r.obs.errorFramesSent.Add(1)
return nil
}
func (r *recordStream) writePayloadFrame(payload []byte) error {
@@ -972,3 +1084,13 @@ func (r *recordStream) signalStateLocked() {
close(r.stateNotify)
r.stateNotify = make(chan struct{})
}
func (r *recordStream) updatePendingApplyLocked() {
if r == nil {
return
}
pending := recordPendingCount(r.inboundReceivedSeq, r.inboundAppliedSeq)
if pending > r.maxPendingApply {
r.maxPendingApply = pending
}
}