fix(notify): 修复传输生命周期竞态，完善背压与协议边界

- 完善 stream/bulk DataID 分配、预留和双向命名空间，修复并发打开及 dedicated/shared 回退时的 ID 冲突 - 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接，防止重连后的旧消息误操作新连接 - 加强 close/reset 身份校验及实例移除检查，修复 dedicated attach 失败、通道引用和资源回收竞态 - 收紧批量发送器停止准入，确保在途入队完成后统一清理请求、缓冲区和等待者 - 修复 record 满队列死锁、取消时序号消耗及关闭竞态，确保关闭有界并返回真实错误 - 增加协商式 record 逻辑半关闭，保留反向 ACK；通过 reset 传递 RecordFailure，避免背压掩盖原始失败原因 - 补齐帧长度、批次数量、序号溢出和未确认窗口校验，提前拒绝超限数据并按字节预算拆批 - 为入站分发增加全局及单连接的条数、字节预算和阻塞背压，关闭时唤醒等待者，消除正常断连日志噪音 - 完善 bulk 窗口释放失败处理与传输诊断，补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
@@ -0,0 +1,279 @@
+package notify
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+)
+
+const (
+	bulkRecoveryQueueSize = 64
+	bulkRecoveryWorkers   = 4
+	bulkRecoveryAttempts  = 3
+	bulkRecoveryAttempt   = 750 * time.Millisecond
+	bulkRecoveryBackoff   = 50 * time.Millisecond
+)
+
+var errBulkRecoveryQueueFull = errors.New("bulk recovery queue full")
+
+type bulkRecoveryTask func(context.Context) error
+
+// bulkRecoveryQueue bounds cleanup work after failed bulk opens. A small fixed
+// worker set prevents both unbounded goroutines and multi-minute serial drain.
+type bulkRecoveryQueue struct {
+	mu             sync.Mutex
+	tasks          []bulkRecoveryTask
+	workerRunning  bool
+	workersRunning int
+	onError        func(error)
+}
+
+func newBulkRecoveryQueue(onError func(error)) *bulkRecoveryQueue {
+	return &bulkRecoveryQueue{
+		onError: onError,
+	}
+}
+
+func (q *bulkRecoveryQueue) enqueue(task bulkRecoveryTask) bool {
+	if task == nil {
+		return true
+	}
+	if q == nil {
+		return false
+	}
+	q.mu.Lock()
+	if len(q.tasks) >= bulkRecoveryQueueSize {
+		q.mu.Unlock()
+		return false
+	}
+	q.tasks = append(q.tasks, task)
+	start := 0
+	for q.workersRunning < bulkRecoveryWorkers && q.workersRunning < len(q.tasks) {
+		q.workersRunning++
+		start++
+	}
+	q.workerRunning = q.workersRunning > 0
+	q.mu.Unlock()
+	for i := 0; i < start; i++ {
+		go q.loop()
+	}
+	return true
+}
+
+func (q *bulkRecoveryQueue) loop() {
+	for {
+		q.mu.Lock()
+		if len(q.tasks) == 0 {
+			q.workersRunning--
+			q.workerRunning = q.workersRunning > 0
+			q.mu.Unlock()
+			return
+		}
+		task := q.tasks[0]
+		copy(q.tasks, q.tasks[1:])
+		q.tasks[len(q.tasks)-1] = nil
+		q.tasks = q.tasks[:len(q.tasks)-1]
+		q.mu.Unlock()
+		q.execute(task)
+	}
+}
+
+func (q *bulkRecoveryQueue) execute(task bulkRecoveryTask) {
+	if q == nil || task == nil {
+		return
+	}
+	q.run(task)
+}
+
+func (q *bulkRecoveryQueue) run(task bulkRecoveryTask) {
+	if q == nil || task == nil {
+		return
+	}
+	lastErr := runBulkRecoveryTaskContext(context.Background(), task)
+	if lastErr != nil && q.onError != nil {
+		q.onError(lastErr)
+	}
+}
+
+// runBulkRecoveryTask performs a bounded reset attempt synchronously. Callers
+// that must establish ordering (for example, Auto dedicated -> shared fallback)
+// use this path so a later open cannot overtake cleanup queued in the background.
+func runBulkRecoveryTask(task bulkRecoveryTask) error {
+	return runBulkRecoveryTaskContext(context.Background(), task)
+}
+
+func runBulkRecoveryTaskContext(parent context.Context, task bulkRecoveryTask) error {
+	if task == nil {
+		return nil
+	}
+	if parent == nil {
+		parent = context.Background()
+	}
+	deadline := time.Now().Add(bulkOpenRecoveryTimeout)
+	var lastErr error
+	for attempt := 0; attempt < bulkRecoveryAttempts; attempt++ {
+		if err := parent.Err(); err != nil {
+			return err
+		}
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		attemptTimeout := remaining
+		if attemptTimeout > bulkRecoveryAttempt {
+			attemptTimeout = bulkRecoveryAttempt
+		}
+		ctx, cancel := context.WithTimeout(parent, attemptTimeout)
+		err := task(ctx)
+		cancel()
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if !bulkRecoveryErrorRetryable(err) {
+			break
+		}
+		if attempt+1 >= bulkRecoveryAttempts {
+			break
+		}
+		backoff := bulkRecoveryBackoff << attempt
+		if backoff > time.Until(deadline) {
+			backoff = time.Until(deadline)
+		}
+		if backoff > 0 {
+			timer := time.NewTimer(backoff)
+			select {
+			case <-parent.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return parent.Err()
+			case <-timer.C:
+			}
+		}
+	}
+	return lastErr
+}
+
+func bulkRecoveryErrorRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	return !errors.Is(err, errTransportDetached) &&
+		!errors.Is(err, errServiceShutdown) &&
+		!errors.Is(err, net.ErrClosed) &&
+		!errors.Is(err, io.ErrClosedPipe)
+}
+
+func (c *ClientCommon) resetBulkAtRouteAndWait(ctx context.Context, route clientSessionRoute, req BulkResetRequest) error {
+	if c == nil {
+		return errBulkClientNil
+	}
+	err := runBulkRecoveryTaskContext(ctx, newClientBulkResetRecoveryTaskAtRoute(c, route, req))
+	if err != nil && bulkRecoveryErrorRetryable(err) {
+		c.bestEffortBulkResetAtRoute(route, req)
+	}
+	return err
+}
+
+func (c *ClientCommon) cleanupBulkResetAtRoute(ctx context.Context, route clientSessionRoute, req BulkResetRequest, wait bool) error {
+	if wait {
+		return c.resetBulkAtRouteAndWait(ctx, route, req)
+	}
+	c.bestEffortBulkResetAtRoute(route, req)
+	return nil
+}
+
+func (s *ServerCommon) resetBulkLogicalAndWait(ctx context.Context, logical *LogicalConn, transport *TransportConn, req BulkResetRequest) error {
+	if s == nil {
+		return errBulkServerNil
+	}
+	err := runBulkRecoveryTaskContext(ctx, newServerBulkResetRecoveryTask(s, logical, transport, req))
+	if err != nil && bulkRecoveryErrorRetryable(err) {
+		s.bestEffortBulkResetLogical(logical, transport, req)
+	}
+	return err
+}
+
+func (s *ServerCommon) cleanupBulkLogicalReset(ctx context.Context, logical *LogicalConn, transport *TransportConn, req BulkResetRequest, wait bool) error {
+	if wait {
+		return s.resetBulkLogicalAndWait(ctx, logical, transport, req)
+	}
+	s.bestEffortBulkResetLogical(logical, transport, req)
+	return nil
+}
+
+func (s *ServerCommon) resetBulkTransportAndWait(ctx context.Context, transport *TransportConn, req BulkResetRequest) error {
+	if s == nil {
+		return errBulkServerNil
+	}
+	err := runBulkRecoveryTaskContext(ctx, newServerBulkResetRecoveryTask(s, transport.logicalConnSnapshot(), transport, req))
+	if err != nil && bulkRecoveryErrorRetryable(err) {
+		s.bestEffortBulkResetTransport(transport, req)
+	}
+	return err
+}
+
+func (s *ServerCommon) cleanupBulkTransportReset(ctx context.Context, transport *TransportConn, req BulkResetRequest, wait bool) error {
+	if wait {
+		return s.resetBulkTransportAndWait(ctx, transport, req)
+	}
+	s.bestEffortBulkResetTransport(transport, req)
+	return nil
+}
+
+func (c *ClientCommon) handleBulkRecoveryOverflow(epoch uint64, req BulkResetRequest) {
+	route := c.clientSessionRouteSnapshot()
+	route.epoch = epoch
+	c.handleBulkRecoveryOverflowAtRoute(route, req)
+}
+
+func (c *ClientCommon) handleBulkRecoveryOverflowAtRoute(route clientSessionRoute, req BulkResetRequest) {
+	if c == nil {
+		return
+	}
+	err := fmt.Errorf("%w: bulk=%s data=%d", errBulkRecoveryQueueFull, req.BulkID, req.DataID)
+	c.reportBulkRecoveryError(err)
+	if c.clientSessionRouteCurrent(route) && route.epoch != 0 {
+		c.stopClientSessionIfCurrent(route.epoch, "bulk recovery queue full", err)
+	}
+}
+
+func (s *ServerCommon) handleBulkRecoveryOverflow(logical *LogicalConn, transport *TransportConn, req BulkResetRequest) {
+	if s == nil {
+		return
+	}
+	err := fmt.Errorf("%w: bulk=%s data=%d", errBulkRecoveryQueueFull, req.BulkID, req.DataID)
+	s.reportBulkRecoveryError(err)
+	if logical != nil && transport != nil && transport.IsCurrent() {
+		s.detachLogicalSessionTransport(logical, "bulk recovery queue full", err)
+	}
+}
+
+func (c *ClientCommon) reportBulkRecoveryError(err error) {
+	if c == nil || err == nil {
+		return
+	}
+	c.mu.Lock()
+	debug := c.showError || c.debugMode
+	c.mu.Unlock()
+	if debug {
+		fmt.Printf("notify bulk reset recovery failed: %v\n", err)
+	}
+}
+
+func (s *ServerCommon) reportBulkRecoveryError(err error) {
+	if s == nil || err == nil {
+		return
+	}
+	s.mu.RLock()
+	debug := s.showError || s.debugMode
+	s.mu.RUnlock()
+	if debug {
+		fmt.Printf("notify bulk reset recovery failed: %v\n", err)
+	}
+}