b612/notify
1
0
Fork 0
Code Issues Pull Requests Projects Releases 12 Wiki Activity

fix(notify): 修复传输生命周期竞态，完善背压与协议边界

Browse Source 
- 完善 stream/bulk DataID 分配、预留和双向命名空间，修复并发打开及 dedicated/shared 回退时的 ID 冲突
- 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接，防止重连后的旧消息误操作新连接
- 加强 close/reset 身份校验及实例移除检查，修复 dedicated attach 失败、通道引用和资源回收竞态
- 收紧批量发送器停止准入，确保在途入队完成后统一清理请求、缓冲区和等待者
- 修复 record 满队列死锁、取消时序号消耗及关闭竞态，确保关闭有界并返回真实错误
- 增加协商式 record 逻辑半关闭，保留反向 ACK；通过 reset 传递 RecordFailure，避免背压掩盖原始失败原因
- 补齐帧长度、批次数量、序号溢出和未确认窗口校验，提前拒绝超限数据并按字节预算拆批
- 为入站分发增加全局及单连接的条数、字节预算和阻塞背压，关闭时唤醒等待者，消除正常断连日志噪音
- 完善 bulk 窗口释放失败处理与传输诊断，补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
This commit is contained in:
兔子
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files
record_stream.go
+178 -174
View File
@@ -27,6 +27,9 @@ const (
defaultRecordInboundQueueLimit = 128
defaultRecordAckEveryRecords = 64
defaultRecordAckDelay = time.Millisecond
recordMaxBatchRecords = 1<<16 - 1
recordMaxPayloadBytes = transferFrameMaxPayloadBytes - recordFrameHeaderSize - recordBatchHeaderV2Size - 4
recordMaxBatchPayloadBytes = transferFrameMaxPayloadBytes - recordFrameHeaderSize - recordBatchHeaderV2Size - 4*recordMaxBatchRecords
)
type RecordFailure struct {
@@ -100,11 +103,14 @@ type recordConfig struct {
InboundQueueLimit int
AckEveryRecords int
AckDelay time.Duration
CloseTimeout time.Duration
}
type recordFlushRequest struct {
ctx context.Context
targetSeq uint64
forceAck bool
closeMode recordCloseMode
done chan error
}
@@ -123,18 +129,26 @@ type recordObservability struct {
}
type recordStream struct {
stream Stream
ctx context.Context
cancel context.CancelFunc
cfg recordConfig
writeMu sync.Mutex
sendCh chan recordOutboundMessage
flushCh chan recordFlushRequest
recvCh chan RecordMessage
ackCh chan struct{}
readerCh chan struct{}
useBatchAck bool
obs recordObservability
stream Stream
ctx context.Context
cancel context.CancelFunc
cfg recordConfig
writeMu sync.Mutex
sendCh chan recordOutboundMessage
sendReady chan struct{}
flushCh chan recordFlushRequest
recvCh chan RecordMessage
ackCh chan struct{}
readerCh chan struct{}
writerCh chan struct{}
useBatchAck bool
useHalfClose bool
recvCloseOnce sync.Once
obs recordObservability
closeMu sync.Mutex
closeDone bool
closeErr error
halfClosed bool
mu sync.Mutex
@@ -160,9 +174,10 @@ type recordStream struct {
inboundAckSentSeq uint64
maxPendingApply int
remoteClosed bool
readErr error
terminalErr error
remoteClosed bool
inboundClosed bool
readErr error
terminalErr error
}
var (
@@ -170,6 +185,7 @@ var (
errRecordRuntimeNil = errors.New("record runtime is nil")
errRecordHandlerNotConfigured = errors.New("record handler is not configured")
errRecordWriteClosed = errors.New("record stream write side is closed")
errRecordPayloadTooLarge = errors.New("record payload too large")
errRecordSeqNotReceived = errors.New("record sequence not received")
)
@@ -177,9 +193,15 @@ func normalizeRecordOpenOptions(opt RecordOpenOptions) RecordOpenOptions {
if opt.MaxBatchRecords <= 0 {
opt.MaxBatchRecords = defaultRecordMaxBatchRecords
}
if opt.MaxBatchRecords > recordMaxBatchRecords {
opt.MaxBatchRecords = recordMaxBatchRecords
}
if opt.MaxBatchBytes <= 0 {
opt.MaxBatchBytes = defaultRecordMaxBatchBytes
}
if opt.MaxBatchBytes > recordMaxBatchPayloadBytes {
opt.MaxBatchBytes = recordMaxBatchPayloadBytes
}
if opt.MaxBatchDelay <= 0 {
opt.MaxBatchDelay = defaultRecordMaxBatchDelay
}
@@ -212,6 +234,7 @@ func recordConfigFromOptions(opt RecordOpenOptions) recordConfig {
InboundQueueLimit: opt.InboundQueueLimit,
AckEveryRecords: opt.AckEveryRecords,
AckDelay: opt.AckDelay,
CloseTimeout: opt.Stream.WriteTimeout,
}
}
@@ -232,16 +255,19 @@ func WrapStreamAsRecord(stream Stream, opt RecordOpenOptions) (RecordStream, err
}
ctx, cancel := context.WithCancel(parent)
record := &recordStream{
stream: stream,
ctx: ctx,
cancel: cancel,
cfg: recordConfigFromOptions(opt),
sendCh: make(chan recordOutboundMessage, opt.MaxBatchRecords*2),
flushCh: make(chan recordFlushRequest),
recvCh: make(chan RecordMessage, opt.InboundQueueLimit),
ackCh: make(chan struct{}, 1),
readerCh: make(chan struct{}),
useBatchAck: recordStreamUseBatchAck(stream.Metadata()),
stream: stream,
ctx: ctx,
cancel: cancel,
cfg: recordConfigFromOptions(opt),
sendCh: make(chan recordOutboundMessage, opt.MaxBatchRecords*2),
sendReady: make(chan struct{}, 1),
flushCh: make(chan recordFlushRequest),
recvCh: make(chan RecordMessage, opt.InboundQueueLimit),
ackCh: make(chan struct{}, 1),
readerCh: make(chan struct{}),
writerCh: make(chan struct{}),
useBatchAck: recordStreamUseBatchAck(stream.Metadata()),
useHalfClose: recordStreamUseHalfClose(stream.Metadata()),
stateNotify: make(chan struct{}),
outstandingSizes: make(map[uint64]int),
@@ -283,6 +309,10 @@ func (r *recordStream) WriteRecord(ctx context.Context, payload []byte) (uint64,
size := len(payload)
for {
r.mu.Lock()
if err := ctx.Err(); err != nil {
r.mu.Unlock()
return 0, err
}
if err := r.streamErrorLocked(); err != nil {
r.mu.Unlock()
return 0, err
@@ -291,7 +321,15 @@ func (r *recordStream) WriteRecord(ctx context.Context, payload []byte) (uint64,
r.mu.Unlock()
return 0, errRecordWriteClosed
}
if r.outstandingRecords >= r.cfg.MaxUnackedRecords || r.outstandingBytes+size > r.cfg.MaxUnackedBytes {
if size > recordMaxPayloadBytes {
r.mu.Unlock()
return 0, fmt.Errorf("%w: size=%d max_record_payload=%d", errRecordPayloadTooLarge, size, recordMaxPayloadBytes)
}
if size > r.cfg.MaxUnackedBytes {
r.mu.Unlock()
return 0, fmt.Errorf("%w: size=%d max_unacked_bytes=%d", errRecordPayloadTooLarge, size, r.cfg.MaxUnackedBytes)
}
if r.outstandingRecords >= r.cfg.MaxUnackedRecords || size > r.cfg.MaxUnackedBytes-r.outstandingBytes || len(r.sendCh) == cap(r.sendCh) {
wait := r.stateNotify
r.mu.Unlock()
select {
@@ -300,9 +338,14 @@ func (r *recordStream) WriteRecord(ctx context.Context, payload []byte) (uint64,
case <-ctx.Done():
return 0, ctx.Err()
case <-wait:
case <-r.sendReady:
}
continue
}
if r.nextOutboundSeq == ^uint64(0) {
r.mu.Unlock()
return 0, errRecordSeqInvalid
}
r.nextOutboundSeq++
msg := recordOutboundMessage{
Seq: r.nextOutboundSeq,
@@ -311,22 +354,12 @@ func (r *recordStream) WriteRecord(ctx context.Context, payload []byte) (uint64,
r.outstandingRecords++
r.outstandingBytes += size
r.outstandingSizes[msg.Seq] = size
select {
case <-r.ctx.Done():
r.rollbackReservedOutboundLocked(msg.Seq)
err := r.streamErrorLocked()
r.mu.Unlock()
return 0, err
case <-ctx.Done():
r.rollbackReservedOutboundLocked(msg.Seq)
r.mu.Unlock()
return 0, ctx.Err()
case r.sendCh <- msg:
r.enqueuedOutboundSeq = msg.Seq
r.signalStateLocked()
r.mu.Unlock()
return msg.Seq, nil
}
// All producers hold mu; only the consumer can change the checked capacity.
r.sendCh <- msg
r.enqueuedOutboundSeq = msg.Seq
r.signalStateLocked()
r.mu.Unlock()
return msg.Seq, nil
}
}
@@ -341,6 +374,7 @@ func (r *recordStream) Flush(ctx context.Context) error {
return err
}
req := recordFlushRequest{
ctx: ctx,
targetSeq: r.flushTargetSeq(),
done: make(chan error, 1),
}
@@ -474,44 +508,34 @@ func (r *recordStream) FailRecord(seq uint64, failure RecordFailure) error {
if failure.Code == "" {
failure.Code = RecordErrorCodeApplyFailed
}
err := r.sendFailureFrame(failure)
if err != nil {
return err
}
r.setTerminalError(failure)
return r.stream.Reset(failure)
return r.notifyFailureAndAbort(failure)
}
func (r *recordStream) CloseWrite() error {
if r == nil {
return errRecordStreamNil
func (r *recordStream) notifyFailureAndAbort(failure RecordFailure) error {
ctx, cancel := context.WithTimeout(context.Background(), r.abortTimeout())
defer cancel()
deadline, _ := ctx.Deadline()
_ = r.stream.SetWriteDeadline(deadline)
done := make(chan error, 1)
go func() { done <- r.sendFailureFrame(failure) }()
var sendErr error
select {
case sendErr = <-done:
case <-ctx.Done():
sendErr = ctx.Err()
}
if err := r.Flush(context.Background()); err != nil {
return err
}
if err := r.flushAckNow(); err != nil {
return err
}
r.mu.Lock()
r.outboundClosed = true
r.signalStateLocked()
r.mu.Unlock()
return r.stream.CloseWrite()
}
func (r *recordStream) Close() error {
if r == nil {
return nil
}
_ = r.flushAckNow()
r.cancel()
return r.stream.Close()
r.abortUnderlyingStream(failure)
return sendErr
}
func (r *recordStream) Reset(err error) error {
if r == nil {
return nil
}
if err == nil {
err = io.ErrClosedPipe
}
r.setTerminalError(err)
return r.stream.Reset(err)
}
@@ -553,6 +577,7 @@ func (r *recordStream) waitAckedAtLeast(ctx context.Context, target uint64) erro
}
func (r *recordStream) writerLoop() {
defer close(r.writerCh)
var (
batch []recordOutboundMessage
batches int
@@ -586,6 +611,8 @@ func (r *recordStream) writerLoop() {
}
ackTimerCh = nil
}
defer stopBatchTimer()
defer stopAckTimer()
scheduleAck := func(hasPendingBatch bool, force bool) (uint64, bool) {
ackSeq := r.pendingAckSeq()
if ackSeq == 0 {
@@ -654,6 +681,23 @@ func (r *recordStream) writerLoop() {
}
return nil
}
appendBatch := func(req recordOutboundMessage) (bool, error) {
if len(batch) > 0 && (batches >= r.cfg.MaxBatchRecords || bytes+len(req.Payload) > r.cfg.MaxBatchBytes) {
if err := flushBatch(); err != nil {
return false, err
}
}
batch = append(batch, req)
batches++
bytes += len(req.Payload)
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flushBatch(); err != nil {
return false, err
}
return true, nil
}
return false, nil
}
flushUntil := func(target uint64) error {
for {
if target == 0 {
@@ -675,13 +719,8 @@ func (r *recordStream) writerLoop() {
if !ok {
return r.streamError()
}
batch = append(batch, req)
batches++
bytes += len(req.Payload)
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flushBatch(); err != nil {
return err
}
if _, err := appendBatch(req); err != nil {
return err
}
}
}
@@ -690,9 +729,15 @@ func (r *recordStream) writerLoop() {
case <-r.ctx.Done():
return
case req := <-r.sendCh:
batch = append(batch, req)
batches++
bytes += len(req.Payload)
r.notifyOutboundReady()
flushed, err := appendBatch(req)
if err != nil {
r.abortRecord(err)
return
}
if flushed {
continue
}
if len(batch) == 1 && r.cfg.MaxBatchDelay > 0 {
if batchTimer == nil {
batchTimer = time.NewTimer(r.cfg.MaxBatchDelay)
@@ -701,36 +746,43 @@ func (r *recordStream) writerLoop() {
}
batchTimerCh = batchTimer.C
}
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flushBatch(); err != nil {
r.setTerminalError(err)
return
}
continue
}
if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
r.abortRecord(err)
return
}
}
case req := <-r.flushCh:
if req.ctx != nil && req.ctx.Err() != nil {
req.done <- req.ctx.Err()
continue
}
err := flushUntil(req.targetSeq)
if err == nil && req.forceAck {
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
err = sendStandaloneAck(ackSeq)
}
}
if err == nil && req.closeMode != recordCloseNone {
err = r.closeUnderlyingFromWriter(req)
}
req.done <- err
if err != nil {
r.abortRecord(err)
return
}
if req.closeMode == recordCloseFull {
return
}
case <-batchTimerCh:
if err := flushBatch(); err != nil {
r.setTerminalError(err)
r.abortRecord(err)
return
}
case <-r.ackCh:
if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
r.abortRecord(err)
return
}
}
@@ -738,7 +790,7 @@ func (r *recordStream) writerLoop() {
stopAckTimer()
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
r.abortRecord(err)
return
}
}
@@ -747,7 +799,7 @@ func (r *recordStream) writerLoop() {
}
func (r *recordStream) readLoop() {
defer close(r.recvCh)
defer r.closeReceive()
defer close(r.readerCh)
for {
payload, err := readTransferFrame(r.stream)
@@ -756,18 +808,12 @@ func (r *recordStream) readLoop() {
r.markRemoteClosed(nil)
return
}
r.setReadError(err)
r.abortRecord(err)
return
}
frame, err := decodeRecordFrame(payload)
if err != nil {
_ = r.sendFailureFrame(RecordFailure{
FailedSeq: r.nextInboundFailureSeq(),
Code: RecordErrorCodeProtocol,
Message: err.Error(),
})
r.setReadError(err)
_ = r.stream.Reset(err)
r.abortProtocol(err)
return
}
switch frame.Type {
@@ -776,34 +822,31 @@ func (r *recordStream) readLoop() {
if frame.AckSeq != 0 {
r.obs.piggybackAckReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err)
_ = r.stream.Reset(err)
r.abortProtocol(err)
return
}
}
if err := r.handleBatchFrame(frame.Batch); err != nil {
_ = r.sendFailureFrame(RecordFailure{
FailedSeq: r.nextInboundFailureSeq(),
Code: RecordErrorCodeProtocol,
Message: err.Error(),
})
r.setReadError(err)
_ = r.stream.Reset(err)
r.abortProtocol(err)
return
}
case recordFrameTypeAck:
r.obs.ackFramesReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err)
_ = r.stream.Reset(err)
r.abortProtocol(err)
return
}
case recordFrameTypeError:
r.obs.errorFramesReceived.Add(1)
r.setReadError(frame.Failure)
r.abortRecord(frame.Failure)
return
case recordFrameTypeFIN:
if err := r.receiveFIN(frame.FinalSeq); err != nil {
r.abortProtocol(err)
return
}
default:
r.setReadError(errRecordFrameInvalid)
r.abortProtocol(errRecordFrameInvalid)
return
}
}
@@ -815,7 +858,7 @@ func (r *recordStream) handleBatchFrame(batch []recordOutboundMessage) error {
}
r.mu.Lock()
expected := r.inboundReceivedSeq + 1
if batch[0].Seq != expected {
if r.inboundClosed || batch[0].Seq != expected {
r.mu.Unlock()
return errRecordSeqInvalid
}
@@ -889,19 +932,6 @@ func (r *recordStream) markRemoteClosed(err error) {
r.mu.Unlock()
}
func (r *recordStream) setReadError(err error) {
if err == nil {
return
}
r.mu.Lock()
if r.readErr == nil {
r.readErr = err
}
r.signalStateLocked()
r.mu.Unlock()
r.cancel()
}
func (r *recordStream) setTerminalError(err error) {
if err == nil {
return
@@ -918,38 +948,14 @@ func (r *recordStream) setTerminalError(err error) {
r.cancel()
}
func (r *recordStream) rollbackReservedOutboundLocked(seq uint64) {
if r == nil || seq == 0 {
return
}
if size, ok := r.outstandingSizes[seq]; ok {
delete(r.outstandingSizes, seq)
r.outstandingBytes -= size
if r.outstandingBytes < 0 {
r.outstandingBytes = 0
}
r.outstandingRecords--
if r.outstandingRecords < 0 {
r.outstandingRecords = 0
}
}
if r.nextOutboundSeq == seq {
r.nextOutboundSeq--
}
r.signalStateLocked()
}
func (r *recordStream) readError() error {
if r == nil {
return errRecordStreamNil
}
r.mu.Lock()
defer r.mu.Unlock()
if r.readErr != nil {
return r.readErr
}
if r.terminalErr != nil {
return r.terminalErr
if err := r.streamErrorLocked(); err != nil {
return err
}
return io.EOF
}
@@ -970,6 +976,14 @@ func (r *recordStream) streamErrorLocked() error {
if r.terminalErr != nil {
return r.terminalErr
}
if stream, ok := r.stream.(*streamHandle); ok && r.ctx.Err() != nil {
if err := stream.resetErrSnapshot(); err != nil {
return err
}
}
if r.ctx != nil {
return r.ctx.Err()
}
return nil
}
@@ -1003,27 +1017,6 @@ func (r *recordStream) markAckSent(ackSeq uint64) {
r.mu.Unlock()
}
func (r *recordStream) flushAckNow() error {
if r == nil {
return errRecordStreamNil
}
req := recordFlushRequest{
forceAck: true,
done: make(chan error, 1),
}
select {
case <-r.ctx.Done():
return r.streamError()
case r.flushCh <- req:
}
select {
case <-r.ctx.Done():
return r.streamError()
case err := <-req.done:
return err
}
}
func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
payload, err := encodeRecordErrorFrame(failure)
if err != nil {
@@ -1043,6 +1036,9 @@ func (r *recordStream) writePayloadFrame(payload []byte) error {
if payload == nil {
return nil
}
if len(payload) > transferFrameMaxPayloadBytes {
return fmt.Errorf("%w: payload=%d max=%d", errTransferFrameTooLarge, len(payload), transferFrameMaxPayloadBytes)
}
frame := buildTransferFrame(payload)
r.writeMu.Lock()
defer r.writeMu.Unlock()
@@ -1067,10 +1063,18 @@ func (r *recordStream) nextOutboundForFlush() (recordOutboundMessage, bool) {
case <-r.ctx.Done():
return recordOutboundMessage{}, false
case req := <-r.sendCh:
r.notifyOutboundReady()
return req, true
}
}
func (r *recordStream) notifyOutboundReady() {
select {
case r.sendReady <- struct{}{}:
default:
}
}
func (r *recordStream) nextInboundFailureSeq() uint64 {
if r == nil {
return 1