fix(notify): 修复传输生命周期竞态,完善背压与协议边界

- 完善 stream/bulk DataID 分配、预留和双向命名空间,修复并发打开及 dedicated/shared 回退时的 ID 冲突
- 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接,防止重连后的旧消息误操作新连接
- 加强 close/reset 身份校验及实例移除检查,修复 dedicated attach 失败、通道引用和资源回收竞态
- 收紧批量发送器停止准入,确保在途入队完成后统一清理请求、缓冲区和等待者
- 修复 record 满队列死锁、取消时序号消耗及关闭竞态,确保关闭有界并返回真实错误
- 增加协商式 record 逻辑半关闭,保留反向 ACK;通过 reset 传递 RecordFailure,避免背压掩盖原始失败原因
- 补齐帧长度、批次数量、序号溢出和未确认窗口校验,提前拒绝超限数据并按字节预算拆批
- 为入站分发增加全局及单连接的条数、字节预算和阻塞背压,关闭时唤醒等待者,消除正常断连日志噪音
- 完善 bulk 窗口释放失败处理与传输诊断,补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
This commit is contained in:
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
+201 -92
View File
@@ -18,20 +18,20 @@ func (s *ServerCommon) OpenBulkLogical(ctx context.Context, logical *LogicalConn
switch opt.Mode {
case BulkOpenModeDedicated:
opt.Dedicated = true
return s.openBulkLogicalWithMode(ctx, logical, opt)
return s.openBulkLogicalWithMode(ctx, logical, opt, false)
case BulkOpenModeAuto:
if err := logicalDedicatedBulkSupportError(logical); err == nil {
dedicatedOpt := opt
dedicatedOpt.Mode = BulkOpenModeDedicated
dedicatedOpt.Dedicated = true
bulk, dedicatedErr := s.openBulkLogicalWithMode(ctx, logical, dedicatedOpt)
bulk, dedicatedErr := s.openBulkLogicalWithMode(ctx, logical, dedicatedOpt, true)
if dedicatedErr == nil {
return bulk, nil
}
sharedOpt := opt
sharedOpt.Mode = BulkOpenModeShared
sharedOpt.Dedicated = false
sharedBulk, sharedErr := s.openBulkLogicalWithMode(ctx, logical, sharedOpt)
sharedBulk, sharedErr := s.openBulkLogicalWithMode(ctx, logical, sharedOpt, false)
if sharedErr == nil {
return sharedBulk, nil
}
@@ -39,19 +39,19 @@ func (s *ServerCommon) OpenBulkLogical(ctx context.Context, logical *LogicalConn
}
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkLogicalWithMode(ctx, logical, opt)
return s.openBulkLogicalWithMode(ctx, logical, opt, false)
case BulkOpenModeShared, BulkOpenModeDefault:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkLogicalWithMode(ctx, logical, opt)
return s.openBulkLogicalWithMode(ctx, logical, opt, false)
default:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkLogicalWithMode(ctx, logical, opt)
return s.openBulkLogicalWithMode(ctx, logical, opt, false)
}
}
func (s *ServerCommon) openBulkLogicalWithMode(ctx context.Context, logical *LogicalConn, opt BulkOpenOptions) (Bulk, error) {
func (s *ServerCommon) openBulkLogicalWithMode(ctx context.Context, logical *LogicalConn, opt BulkOpenOptions, waitForReset bool) (Bulk, error) {
if s == nil {
return nil, errBulkServerNil
}
@@ -59,6 +59,7 @@ func (s *ServerCommon) openBulkLogicalWithMode(ctx context.Context, logical *Log
if logical == nil {
return nil, errBulkLogicalConnNil
}
transport := logical.CurrentTransportConn()
runtime := s.getBulkRuntime()
if runtime == nil {
return nil, errBulkRuntimeNil
@@ -76,83 +77,111 @@ func (s *ServerCommon) openBulkLogicalWithMode(ctx context.Context, logical *Log
if _, exists := runtime.lookup(scope, req.BulkID); exists {
return nil, errBulkAlreadyExists
}
if req.Dedicated {
if req.DataID == 0 {
req.DataID = runtime.nextDataID()
if req.DataID == 0 {
var reserveErr error
req.DataID, reserveErr = runtime.reserveDataID(scope, 0)
if reserveErr != nil {
return nil, reserveErr
}
}
if req.Dedicated {
if req.AttachToken == "" {
req.AttachToken = newBulkAttachToken()
}
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, logical.CurrentTransportConn(), logical.transportGenerationSnapshot(), serverBulkCloseSender(s, logical, nil), serverBulkResetSender(s, logical, nil), serverBulkDataSender(s, logical.CurrentTransportConn()), serverBulkWriteSender(s, logical, logical.CurrentTransportConn()), serverBulkReleaseSender(s, logical, logical.CurrentTransportConn()))
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, logical.transportGenerationSnapshot(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.register(scope, bulk); err != nil {
if err := runtime.adoptReserved(scope, bulk); err != nil {
runtime.releaseDataID(scope, req.DataID)
return nil, err
}
s.attachServerDedicatedSidecarIfExists(logical, bulk)
resp, err := sendBulkOpenServerLogical(ctx, s, logical, req)
resp, err := sendBulkOpenServerTransport(ctx, s, transport, req)
if err != nil {
runtime.releaseDataID(scope, req.DataID)
cleanupErr := s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
_, _ = sendBulkResetServerLogical(context.Background(), s, logical, BulkResetRequest{
cleanupErr := s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: "bulk dedicated data id mismatch",
})
}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.TransportGeneration != 0 {
bulk.transportGeneration = resp.TransportGeneration
bulk.setTransportGeneration(resp.TransportGeneration)
}
if resp.FastPathVersion != 0 {
bulk.fastPathVersion = normalizeBulkFastPathVersion(resp.FastPathVersion)
bulk.setFastPathVersion(resp.FastPathVersion)
}
if resp.AttachToken != "" {
bulk.setDedicatedAttachToken(resp.AttachToken)
}
if err := bulk.waitAcceptReady(ctx); err != nil {
_, _ = sendBulkResetServerLogical(context.Background(), s, logical, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
var cleanupErr error
if bulk.resetErrSnapshot() == nil {
cleanupErr = s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
}, waitForReset)
} else {
s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
}
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
return bulk, nil
}
resp, err := sendBulkOpenServerLogical(ctx, s, logical, req)
if err != nil {
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, logical.transportGenerationSnapshot(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.adoptReserved(scope, bulk); err != nil {
runtime.releaseDataID(scope, req.DataID)
return nil, err
}
if resp.DataID != 0 {
req.DataID = resp.DataID
resp, err := sendBulkOpenServerTransport(ctx, s, transport, req)
if err != nil {
s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()})
bulk.markReset(err)
return nil, err
}
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, Error: "bulk data id mismatch"})
bulk.markReset(err)
return nil, err
}
if resp.FastPathVersion != 0 {
req.FastPathVersion = resp.FastPathVersion
bulk.setFastPathVersion(resp.FastPathVersion)
}
req.Dedicated = resp.Dedicated
if resp.AttachToken != "" {
req.AttachToken = resp.AttachToken
}
if req.DataID == 0 {
return nil, errBulkDataIDEmpty
}
transport := logical.CurrentTransportConn()
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverBulkCloseSender(s, logical, nil), serverBulkResetSender(s, logical, nil), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.register(scope, bulk); err != nil {
_, _ = sendBulkResetServerLogical(context.Background(), s, logical, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
if resp.Dedicated {
err = errBulkRejected
s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: "shared bulk upgraded to dedicated"})
bulk.markReset(err)
return nil, err
}
s.attachServerDedicatedSidecarIfExists(logical, bulk)
if resp.AttachToken != "" {
bulk.setDedicatedAttachToken(resp.AttachToken)
}
if resp.TransportGeneration != 0 {
bulk.setTransportGeneration(resp.TransportGeneration)
}
return bulk, nil
}
@@ -161,20 +190,20 @@ func (s *ServerCommon) OpenBulkTransport(ctx context.Context, transport *Transpo
switch opt.Mode {
case BulkOpenModeDedicated:
opt.Dedicated = true
return s.openBulkTransportWithMode(ctx, transport, opt)
return s.openBulkTransportWithMode(ctx, transport, opt, false)
case BulkOpenModeAuto:
if err := transportDedicatedBulkSupportError(transport); err == nil {
dedicatedOpt := opt
dedicatedOpt.Mode = BulkOpenModeDedicated
dedicatedOpt.Dedicated = true
bulk, dedicatedErr := s.openBulkTransportWithMode(ctx, transport, dedicatedOpt)
bulk, dedicatedErr := s.openBulkTransportWithMode(ctx, transport, dedicatedOpt, true)
if dedicatedErr == nil {
return bulk, nil
}
sharedOpt := opt
sharedOpt.Mode = BulkOpenModeShared
sharedOpt.Dedicated = false
sharedBulk, sharedErr := s.openBulkTransportWithMode(ctx, transport, sharedOpt)
sharedBulk, sharedErr := s.openBulkTransportWithMode(ctx, transport, sharedOpt, false)
if sharedErr == nil {
return sharedBulk, nil
}
@@ -182,19 +211,19 @@ func (s *ServerCommon) OpenBulkTransport(ctx context.Context, transport *Transpo
}
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkTransportWithMode(ctx, transport, opt)
return s.openBulkTransportWithMode(ctx, transport, opt, false)
case BulkOpenModeShared, BulkOpenModeDefault:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkTransportWithMode(ctx, transport, opt)
return s.openBulkTransportWithMode(ctx, transport, opt, false)
default:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return s.openBulkTransportWithMode(ctx, transport, opt)
return s.openBulkTransportWithMode(ctx, transport, opt, false)
}
}
func (s *ServerCommon) openBulkTransportWithMode(ctx context.Context, transport *TransportConn, opt BulkOpenOptions) (Bulk, error) {
func (s *ServerCommon) openBulkTransportWithMode(ctx context.Context, transport *TransportConn, opt BulkOpenOptions, waitForReset bool) (Bulk, error) {
if s == nil {
return nil, errBulkServerNil
}
@@ -223,85 +252,164 @@ func (s *ServerCommon) openBulkTransportWithMode(ctx context.Context, transport
if _, exists := runtime.lookup(scope, req.BulkID); exists {
return nil, errBulkAlreadyExists
}
if req.Dedicated {
if req.DataID == 0 {
req.DataID = runtime.nextDataID()
if req.DataID == 0 {
var reserveErr error
req.DataID, reserveErr = runtime.reserveDataID(scope, 0)
if reserveErr != nil {
return nil, reserveErr
}
}
if req.Dedicated {
if req.AttachToken == "" {
req.AttachToken = newBulkAttachToken()
}
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, transport.TransportGeneration(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.register(scope, bulk); err != nil {
if err := runtime.adoptReserved(scope, bulk); err != nil {
runtime.releaseDataID(scope, req.DataID)
return nil, err
}
s.attachServerDedicatedSidecarIfExists(logical, bulk)
resp, err := sendBulkOpenServerTransport(ctx, s, transport, req)
if err != nil {
runtime.releaseDataID(scope, req.DataID)
cleanupErr := s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
_, _ = sendBulkResetServerTransport(context.Background(), s, transport, BulkResetRequest{
cleanupErr := s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: "bulk dedicated data id mismatch",
})
}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.TransportGeneration != 0 {
bulk.transportGeneration = resp.TransportGeneration
bulk.setTransportGeneration(resp.TransportGeneration)
}
if resp.FastPathVersion != 0 {
bulk.fastPathVersion = normalizeBulkFastPathVersion(resp.FastPathVersion)
bulk.setFastPathVersion(resp.FastPathVersion)
}
if resp.AttachToken != "" {
bulk.setDedicatedAttachToken(resp.AttachToken)
}
if err := bulk.waitAcceptReady(ctx); err != nil {
_, _ = sendBulkResetServerTransport(context.Background(), s, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
var cleanupErr error
if bulk.resetErrSnapshot() == nil {
cleanupErr = s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
}, waitForReset)
} else {
s.bestEffortBulkResetTransport(transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
}
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
return bulk, nil
}
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, transport.TransportGeneration(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.adoptReserved(scope, bulk); err != nil {
runtime.releaseDataID(scope, req.DataID)
return nil, err
}
resp, err := sendBulkOpenServerTransport(ctx, s, transport, req)
if err != nil {
s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()})
bulk.markReset(err)
return nil, err
}
if resp.DataID != 0 {
req.DataID = resp.DataID
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, Error: "bulk data id mismatch"})
bulk.markReset(err)
return nil, err
}
if resp.FastPathVersion != 0 {
req.FastPathVersion = resp.FastPathVersion
bulk.setFastPathVersion(resp.FastPathVersion)
}
req.Dedicated = resp.Dedicated
if resp.AttachToken != "" {
req.AttachToken = resp.AttachToken
}
if req.DataID == 0 {
return nil, errBulkDataIDEmpty
}
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
bulk.markAcceptHandled()
if err := runtime.register(scope, bulk); err != nil {
_, _ = sendBulkResetServerTransport(context.Background(), s, transport, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
if resp.Dedicated {
err = errBulkRejected
s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: "shared bulk upgraded to dedicated"})
bulk.markReset(err)
return nil, err
}
s.attachServerDedicatedSidecarIfExists(logical, bulk)
if resp.AttachToken != "" {
bulk.setDedicatedAttachToken(resp.AttachToken)
}
if resp.TransportGeneration != 0 {
bulk.setTransportGeneration(resp.TransportGeneration)
}
return bulk, nil
}
func (s *ServerCommon) bestEffortBulkResetLogical(logical *LogicalConn, transport *TransportConn, req BulkResetRequest) {
if s == nil || logical == nil {
return
}
task := newServerBulkResetRecoveryTask(s, logical, transport, req)
q := s.bulkRecoveryQueue()
if !q.enqueue(task) {
s.handleBulkRecoveryOverflow(logical, transport, req)
}
}
func (s *ServerCommon) bulkRecoveryQueue() *bulkRecoveryQueue {
if s == nil {
return nil
}
s.bulkRecoveryMu.Lock()
defer s.bulkRecoveryMu.Unlock()
if s.bulkRecovery == nil {
s.bulkRecovery = newBulkRecoveryQueue(s.reportBulkRecoveryError)
}
return s.bulkRecovery
}
func (s *ServerCommon) bestEffortBulkResetTransport(transport *TransportConn, req BulkResetRequest) {
if s == nil || transport == nil {
return
}
task := newServerBulkResetRecoveryTask(s, transport.logicalConnSnapshot(), transport, req)
q := s.bulkRecoveryQueue()
if !q.enqueue(task) {
s.handleBulkRecoveryOverflow(transport.logicalConnSnapshot(), transport, req)
}
}
func newServerBulkResetRecoveryTask(s *ServerCommon, logical *LogicalConn, transport *TransportConn, req BulkResetRequest) bulkRecoveryTask {
return func(ctx context.Context) error {
if transport == nil {
if logical != nil {
return transportDetachedErrorForLogical(logical)
}
return errTransportDetached
}
_, err := sendBulkResetServerTransport(ctx, s, transport, req)
if errors.Is(err, errBulkNotFound) {
return nil
}
return err
}
}
func serverBulkRequest(runtime *bulkRuntime, opt BulkOpenOptions) BulkOpenRequest {
opt = normalizeBulkOpenOptions(opt)
id := opt.ID
@@ -332,13 +440,14 @@ func serverBulkCloseSender(s *ServerCommon, logical *LogicalConn, transport *Tra
}
req := BulkCloseRequest{
BulkID: bulk.ID(),
DataID: bulk.dataIDSnapshot(),
Full: full,
}
if logical != nil {
_, err := sendBulkCloseServerLogical(ctx, s, logical, req)
if transport != nil {
_, err := sendBulkCloseServerTransport(ctx, s, transport, req)
return err
}
_, err := sendBulkCloseServerTransport(ctx, s, transport, req)
_, err := sendBulkCloseServerLogical(ctx, s, logical, req)
return err
}
}
@@ -356,11 +465,11 @@ func serverBulkResetSender(s *ServerCommon, logical *LogicalConn, transport *Tra
DataID: bulk.dataIDSnapshot(),
Error: message,
}
if logical != nil {
_, err := sendBulkResetServerLogical(ctx, s, logical, req)
if transport != nil {
_, err := sendBulkResetServerTransport(ctx, s, transport, req)
return err
}
_, err := sendBulkResetServerTransport(ctx, s, transport, req)
_, err := sendBulkResetServerLogical(ctx, s, logical, req)
return err
}
}
@@ -461,7 +570,7 @@ func serverBulkReleaseSender(s *ServerCommon, logical *LogicalConn, transport *T
Bytes: bytes,
Chunks: chunks,
}
if transport != nil && transport.IsCurrent() {
if transport != nil {
return sendBulkReleaseServerTransport(ctx, s, transport, req)
}
return sendBulkReleaseServerLogical(ctx, s, logical, req)