b612/notify
1
0
Fork 0
Code Issues Pull Requests Projects Releases 12 Wiki Activity

fix(notify): 修复传输生命周期竞态，完善背压与协议边界

Browse Source 
- 完善 stream/bulk DataID 分配、预留和双向命名空间，修复并发打开及 dedicated/shared 回退时的 ID 冲突
- 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接，防止重连后的旧消息误操作新连接
- 加强 close/reset 身份校验及实例移除检查，修复 dedicated attach 失败、通道引用和资源回收竞态
- 收紧批量发送器停止准入，确保在途入队完成后统一清理请求、缓冲区和等待者
- 修复 record 满队列死锁、取消时序号消耗及关闭竞态，确保关闭有界并返回真实错误
- 增加协商式 record 逻辑半关闭，保留反向 ACK；通过 reset 传递 RecordFailure，避免背压掩盖原始失败原因
- 补齐帧长度、批次数量、序号溢出和未确认窗口校验，提前拒绝超限数据并按字节预算拆批
- 为入站分发增加全局及单连接的条数、字节预算和阻塞背压，关闭时唤醒等待者，消除正常断连日志噪音
- 完善 bulk 窗口释放失败处理与传输诊断，补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
This commit is contained in:
兔子
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bulk_dedicated_lane_sender.go
+132 -30
View File
@@ -2,6 +2,7 @@ package notify
import (
"context"
"fmt"
"net"
"sync"
"sync/atomic"
@@ -28,22 +29,27 @@ type bulkDedicatedLaneSender struct {
encode func([]bulkDedicatedOutboundBatch) ([]byte, func(), error)
fail func(error)
reqCh chan *bulkDedicatedLaneBatchRequest
stopCh chan struct{}
doneCh chan struct{}
stopOnce sync.Once
flushMu sync.Mutex
queued atomic.Int64
reqCh chan *bulkDedicatedLaneBatchRequest
stopCh chan struct{}
doneCh chan struct{}
stopOnce sync.Once
admissionMu sync.Mutex
admitting sync.WaitGroup
admissionClosed bool
flushMu sync.Mutex
queued atomic.Int64
debug bool
errMu sync.Mutex
err error
}
func newBulkDedicatedLaneSender(conn net.Conn, encode func([]bulkDedicatedOutboundBatch) ([]byte, func(), error), fail func(error)) *bulkDedicatedLaneSender {
func newBulkDedicatedLaneSender(conn net.Conn, encode func([]bulkDedicatedOutboundBatch) ([]byte, func(), error), fail func(error), debug ...bool) *bulkDedicatedLaneSender {
sender := &bulkDedicatedLaneSender{
conn: conn,
encode: encode,
fail: fail,
debug: len(debug) > 0 && debug[0],
reqCh: make(chan *bulkDedicatedLaneBatchRequest, bulkDedicatedSendQueueSize),
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
@@ -210,7 +216,16 @@ func (s *bulkDedicatedLaneSender) submitControl(ctx context.Context, dataID uint
if len(payload) > 0 {
items[0].Payload = append([]byte(nil), payload...)
}
return s.submitBatch(ctx, dataID, items, true, false)
var startedAt time.Time
if s.debug && frameType == bulkFastPayloadTypeRelease {
startedAt = time.Now()
fmt.Printf("[bulk-debug] at=%s lane release enqueue data=%d queued=%d\n", startedAt.Format(time.RFC3339Nano), dataID, s.queued.Load())
}
err := s.submitBatch(ctx, dataID, items, true, false)
if s.debug && frameType == bulkFastPayloadTypeRelease {
fmt.Printf("[bulk-debug] at=%s lane release done data=%d elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), dataID, time.Since(startedAt), err)
}
return err
}
func (s *bulkDedicatedLaneSender) submitBatch(ctx context.Context, dataID uint64, items []bulkDedicatedSendRequest, wait bool, borrowItems bool) error {
@@ -226,21 +241,18 @@ func (s *bulkDedicatedLaneSender) submitBatch(ctx context.Context, dataID uint64
req := getBulkDedicatedLaneBatchRequest()
req.prepare(ctx, dataID, items, wait, borrowItems)
s.queued.Add(1)
select {
case <-ctx.Done():
if !s.enqueue(req) {
s.queued.Add(-1)
req.recycle()
return normalizeStreamDeadlineError(ctx.Err())
case <-s.stopCh:
s.queued.Add(-1)
req.recycle()
return s.stoppedErr()
case s.reqCh <- req:
if !wait {
return nil
if err := ctx.Err(); err != nil {
return normalizeStreamDeadlineError(err)
}
return s.waitAck(req)
return s.stoppedErr()
}
if !wait {
return nil
}
return s.waitAck(req)
}
func (s *bulkDedicatedLaneSender) tryDirectSubmitWrite(ctx context.Context, dataID uint64, startSeq uint64, payload []byte, chunkSize int) (bool, int, error) {
@@ -325,12 +337,18 @@ func (s *bulkDedicatedLaneSender) tryDirectSubmitWrite(ctx context.Context, data
seq++
written = end
}
select {
case <-s.stopCh:
return true, start, s.stoppedErr()
default:
}
if err := s.flush([]bulkDedicatedOutboundBatch{{
DataID: dataID,
Items: items,
}}, deadline); err != nil {
err = normalizeDedicatedBulkSendError(err)
s.setErr(err)
s.markFailed(err)
s.waitAdmissions()
s.failPending(err)
if s.fail != nil {
go s.fail(err)
@@ -382,12 +400,18 @@ func (s *bulkDedicatedLaneSender) tryDirectSubmitBatch(ctx context.Context, data
default:
}
deadline, _ := ctx.Deadline()
select {
case <-s.stopCh:
return true, s.stoppedErr()
default:
}
if err := s.flush([]bulkDedicatedOutboundBatch{{
DataID: dataID,
Items: items,
}}, deadline); err != nil {
err = normalizeDedicatedBulkSendError(err)
s.setErr(err)
s.markFailed(err)
s.waitAdmissions()
s.failPending(err)
if s.fail != nil {
go s.fail(err)
@@ -427,11 +451,35 @@ func (s *bulkDedicatedLaneSender) stop() {
if s == nil {
return
}
s.stopOnce.Do(func() {
s.setErr(errTransportDetached)
close(s.stopCh)
})
s.markFailed(errTransportDetached)
s.waitAdmissions()
<-s.doneCh
// Direct submissions flush on the caller goroutine rather than run().
// Drain that path before the connection can be handed to a replacement.
s.flushMu.Lock()
s.flushMu.Unlock()
}
func (s *bulkDedicatedLaneSender) enqueue(req *bulkDedicatedLaneBatchRequest) bool {
if s == nil || req == nil {
return false
}
s.admissionMu.Lock()
if s.admissionClosed {
s.admissionMu.Unlock()
return false
}
s.admitting.Add(1)
s.admissionMu.Unlock()
defer s.admitting.Done()
select {
case <-req.Ctx.Done():
return false
case <-s.stopCh:
return false
case s.reqCh <- req:
return true
}
}
func (s *bulkDedicatedLaneSender) run() {
@@ -456,25 +504,42 @@ func (s *bulkDedicatedLaneSender) run() {
DataID: req.DataID,
Items: req.Items,
}}
batchBytes := bulkDedicatedBatchesPlainLen(batches)
batchBytes, err := bulkDedicatedBatchesPlainLenChecked(batches)
if err != nil {
s.finishRequest(req, err)
continue
}
deadline := req.Deadline
var lockStarted time.Time
if s.debug {
lockStarted = time.Now()
}
s.flushMu.Lock()
err := s.errSnapshot()
if s.debug {
fmt.Printf("[bulk-debug] at=%s lane flush acquired data=%d lock-wait=%s queued=%d\n", time.Now().Format(time.RFC3339Nano), req.DataID, time.Since(lockStarted), s.queued.Load())
}
err = s.errSnapshot()
if err == nil {
carry, err = s.collectBatchRequests(&batchReqs, &batches, &batchBytes, &deadline)
if err == nil {
err = s.flush(batches, deadline)
select {
case <-s.stopCh:
err = s.stoppedErr()
default:
err = s.flush(batches, deadline)
}
}
}
s.flushMu.Unlock()
if err != nil {
err = normalizeDedicatedBulkSendError(err)
s.setErr(err)
s.markFailed(err)
s.finishBatchRequests(batchReqs, err)
if carry != nil {
s.finishRequest(carry, err)
carry = nil
}
s.waitAdmissions()
s.failPending(err)
if s.fail != nil {
go s.fail(err)
@@ -566,6 +631,7 @@ func (s *bulkDedicatedLaneSender) nextRequest(carry *bulkDedicatedLaneBatchReque
select {
case <-s.stopCh:
err := s.stoppedErr()
s.waitAdmissions()
s.finishRequest(carry, err)
s.failPending(err)
return nil, false
@@ -575,6 +641,7 @@ func (s *bulkDedicatedLaneSender) nextRequest(carry *bulkDedicatedLaneBatchReque
}
select {
case <-s.stopCh:
s.waitAdmissions()
s.failPending(s.stoppedErr())
return nil, false
case req := <-s.reqCh:
@@ -670,6 +737,10 @@ func (s *bulkDedicatedLaneSender) flush(batches []bulkDedicatedOutboundBatch, de
if s == nil || s.conn == nil {
return errTransportDetached
}
var startedAt time.Time
if s.debug {
startedAt = time.Now()
}
payload, release, err := s.encode(batches)
if err != nil {
return err
@@ -677,7 +748,19 @@ func (s *bulkDedicatedLaneSender) flush(batches []bulkDedicatedOutboundBatch, de
if release != nil {
defer release()
}
return writeBulkDedicatedRecordWithDeadline(s.conn, payload, deadline)
if !s.debug {
return writeBulkDedicatedRecordWithDeadline(s.conn, payload, deadline)
}
encodeElapsed := time.Since(startedAt)
dataID := uint64(0)
if len(batches) > 0 {
dataID = batches[0].DataID
}
fmt.Printf("[bulk-debug] at=%s lane write begin data=%d groups=%d bytes=%d encode=%s\n", time.Now().Format(time.RFC3339Nano), dataID, len(batches), len(payload), encodeElapsed)
writeStarted := time.Now()
err = writeBulkDedicatedRecordWithDeadlineTrace(s.conn, payload, deadline, dataID)
fmt.Printf("[bulk-debug] at=%s lane write end data=%d elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), dataID, time.Since(writeStarted), err)
return err
}
func (s *bulkDedicatedLaneSender) finishRequest(req *bulkDedicatedLaneBatchRequest, err error) {
@@ -726,6 +809,25 @@ func (s *bulkDedicatedLaneSender) setErr(err error) {
s.errMu.Unlock()
}
func (s *bulkDedicatedLaneSender) markFailed(err error) {
if s == nil {
return
}
s.setErr(err)
s.stopOnce.Do(func() {
s.admissionMu.Lock()
s.admissionClosed = true
close(s.stopCh)
s.admissionMu.Unlock()
})
}
func (s *bulkDedicatedLaneSender) waitAdmissions() {
if s != nil {
s.admitting.Wait()
}
}
func (s *bulkDedicatedLaneSender) errSnapshot() error {
if s == nil {
return errTransportDetached