b612/notify
1
0
Fork 0
Code Issues Pull Requests Projects Releases 12 Wiki Activity

fix(notify): 修复传输生命周期竞态，完善背压与协议边界

Browse Source 
- 完善 stream/bulk DataID 分配、预留和双向命名空间，修复并发打开及 dedicated/shared 回退时的 ID 冲突
- 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接，防止重连后的旧消息误操作新连接
- 加强 close/reset 身份校验及实例移除检查，修复 dedicated attach 失败、通道引用和资源回收竞态
- 收紧批量发送器停止准入，确保在途入队完成后统一清理请求、缓冲区和等待者
- 修复 record 满队列死锁、取消时序号消耗及关闭竞态，确保关闭有界并返回真实错误
- 增加协商式 record 逻辑半关闭，保留反向 ACK；通过 reset 传递 RecordFailure，避免背压掩盖原始失败原因
- 补齐帧长度、批次数量、序号溢出和未确认窗口校验，提前拒绝超限数据并按字节预算拆批
- 为入站分发增加全局及单连接的条数、字节预算和阻塞背压，关闭时唤醒等待者，消除正常断连日志噪音
- 完善 bulk 窗口释放失败处理与传输诊断，补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
This commit is contained in:
兔子
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client_bulk.go
+170 -70
View File
@@ -3,8 +3,11 @@ package notify
import (
"context"
"errors"
"time"
)
const bulkOpenRecoveryTimeout = 2 * time.Second
func (c *ClientCommon) SetBulkHandler(fn func(BulkAcceptInfo) error) {
runtime := c.getBulkRuntime()
if runtime == nil {
@@ -33,21 +36,21 @@ func (c *ClientCommon) OpenBulk(ctx context.Context, opt BulkOpenOptions) (Bulk,
switch opt.Mode {
case BulkOpenModeDedicated:
opt.Dedicated = true
return c.openBulkWithDedicatedMode(ctx, opt)
return c.openBulkWithDedicatedMode(ctx, opt, false)
case BulkOpenModeAuto:
// Auto mode prefers dedicated path and falls back to shared if dedicated fails.
if err := clientDedicatedBulkSupportError(c); err == nil {
dedicatedOpt := opt
dedicatedOpt.Mode = BulkOpenModeDedicated
dedicatedOpt.Dedicated = true
bulk, dedicatedErr := c.openBulkWithDedicatedMode(ctx, dedicatedOpt)
bulk, dedicatedErr := c.openBulkWithDedicatedMode(ctx, dedicatedOpt, true)
if dedicatedErr == nil {
return bulk, nil
}
sharedOpt := opt
sharedOpt.Mode = BulkOpenModeShared
sharedOpt.Dedicated = false
sharedBulk, sharedErr := c.openBulkWithDedicatedMode(ctx, sharedOpt)
sharedBulk, sharedErr := c.openBulkWithDedicatedMode(ctx, sharedOpt, false)
if sharedErr == nil {
c.bulkAttachFallbackCount.Add(1)
return sharedBulk, nil
@@ -57,22 +60,26 @@ func (c *ClientCommon) OpenBulk(ctx context.Context, opt BulkOpenOptions) (Bulk,
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
c.bulkAttachFallbackCount.Add(1)
return c.openBulkWithDedicatedMode(ctx, opt)
return c.openBulkWithDedicatedMode(ctx, opt, false)
case BulkOpenModeShared, BulkOpenModeDefault:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return c.openBulkWithDedicatedMode(ctx, opt)
return c.openBulkWithDedicatedMode(ctx, opt, false)
default:
opt.Mode = BulkOpenModeShared
opt.Dedicated = false
return c.openBulkWithDedicatedMode(ctx, opt)
return c.openBulkWithDedicatedMode(ctx, opt, false)
}
}
func (c *ClientCommon) openBulkWithDedicatedMode(ctx context.Context, opt BulkOpenOptions) (Bulk, error) {
func (c *ClientCommon) openBulkWithDedicatedMode(ctx context.Context, opt BulkOpenOptions, waitForReset bool) (Bulk, error) {
if c == nil {
return nil, errBulkClientNil
}
route := c.clientSessionRouteSnapshot()
if err := c.ensureClientSessionRouteSendReady(route); err != nil {
return nil, err
}
opt = applyBulkOpenTuningDefaults(opt, c.bulkOpenTuningSnapshot())
runtime := c.getBulkRuntime()
if runtime == nil {
@@ -90,112 +97,199 @@ func (c *ClientCommon) openBulkWithDedicatedMode(ctx context.Context, opt BulkOp
if !validBulkRange(req.Range) {
return nil, errBulkRangeInvalid
}
if _, exists := runtime.lookup(clientFileScope(), req.BulkID); exists {
return nil, errBulkAlreadyExists
if existing, exists := runtime.lookup(clientFileScope(), req.BulkID); exists {
if existing.acceptsClientSessionRoute(route) {
return nil, errBulkAlreadyExists
}
existing.markReset(errTransportDetached)
}
if req.DataID == 0 {
var reserveErr error
req.DataID, reserveErr = runtime.reserveDataID(clientFileScope(), 0)
if reserveErr != nil {
return nil, reserveErr
}
}
if req.Dedicated {
req.DedicatedLaneID = c.reserveBulkDedicatedLane()
if req.DataID == 0 {
req.DataID = runtime.nextDataID()
var laneErr error
req.DedicatedLaneID, laneErr = c.reserveBulkDedicatedLaneAtRoute(route)
if laneErr != nil {
runtime.releaseDataID(clientFileScope(), req.DataID)
return nil, laneErr
}
if req.AttachToken == "" {
req.AttachToken = newBulkAttachToken()
}
bulk := newBulkHandle(c.clientStopContextSnapshot(), runtime, clientFileScope(), req, c.currentClientSessionEpoch(), nil, nil, 0, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, c.currentClientSessionEpoch()), clientBulkWriteSender(c, c.currentClientSessionEpoch()), clientBulkReleaseSender(c))
bulk := newBulkHandle(clientSessionRouteContext(route), runtime, clientFileScope(), req, route.epoch, nil, nil, 0, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, route), clientBulkWriteSender(c, route), clientBulkReleaseSender(c))
bulk.setClientSnapshotOwner(c)
bulk.setClientSessionRoute(route)
bulk.markAcceptHandled()
if err := runtime.register(clientFileScope(), bulk); err != nil {
c.releaseBulkDedicatedLane(req.DedicatedLaneID)
bulk.markDedicatedLaneReserved()
if err := runtime.adoptReserved(clientFileScope(), bulk); err != nil {
runtime.releaseDataID(clientFileScope(), req.DataID)
return nil, err
}
resp, err := sendBulkOpenClient(ctx, c, req)
resp, err := sendBulkOpenClientAtRoute(ctx, c, route, req)
if err != nil {
runtime.releaseDataID(clientFileScope(), req.DataID)
cleanupErr := c.cleanupBulkResetAtRoute(ctx, route, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
_, _ = sendBulkResetClient(context.Background(), c, BulkResetRequest{
cleanupErr := c.cleanupBulkResetAtRoute(ctx, route, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: "bulk dedicated data id mismatch",
})
}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if resp.TransportGeneration != 0 {
bulk.transportGeneration = resp.TransportGeneration
bulk.setTransportGeneration(resp.TransportGeneration)
}
if resp.FastPathVersion != 0 {
bulk.fastPathVersion = normalizeBulkFastPathVersion(resp.FastPathVersion)
bulk.setFastPathVersion(resp.FastPathVersion)
}
if resp.AttachToken != "" {
req.AttachToken = resp.AttachToken
bulk.setDedicatedAttachToken(resp.AttachToken)
}
if err := c.attachDedicatedBulkSidecar(ctx, bulk); err != nil {
_, _ = sendBulkResetClient(context.Background(), c, BulkResetRequest{
cleanupErr := c.cleanupBulkResetAtRoute(ctx, route, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
}, waitForReset)
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
if err := bulk.waitAcceptReady(ctx); err != nil {
_, _ = sendBulkResetClient(context.Background(), c, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
var cleanupErr error
if bulk.resetErrSnapshot() == nil {
cleanupErr = c.cleanupBulkResetAtRoute(ctx, route, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
}, waitForReset)
} else {
// A ready error already reset the remote handle. Keep the old
// asynchronous cleanup for compatibility with that path and avoid
// racing a concurrent dedicated attach teardown.
c.bestEffortBulkResetAtRoute(route, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
}
bulk.markReset(err)
if cleanupErr != nil {
return nil, errors.Join(err, cleanupErr)
}
return nil, err
}
return bulk, nil
}
resp, err := sendBulkOpenClient(ctx, c, req)
if err != nil {
bulk := newBulkHandle(clientSessionRouteContext(route), runtime, clientFileScope(), req, route.epoch, nil, nil, 0, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, route), clientBulkWriteSender(c, route), clientBulkReleaseSender(c))
bulk.setClientSnapshotOwner(c)
bulk.setClientSessionRoute(route)
bulk.markAcceptHandled()
if err := runtime.adoptReserved(clientFileScope(), bulk); err != nil {
runtime.releaseDataID(clientFileScope(), req.DataID)
return nil, err
}
if resp.DataID != 0 {
req.DataID = resp.DataID
resp, err := sendBulkOpenClientAtRoute(ctx, c, route, req)
if err != nil {
c.bestEffortBulkResetAtRoute(route, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()})
bulk.markReset(err)
return nil, err
}
if resp.DataID != 0 && resp.DataID != req.DataID {
err = errBulkAlreadyExists
c.bestEffortBulkResetAtRoute(route, BulkResetRequest{BulkID: req.BulkID, Error: "bulk data id mismatch"})
bulk.markReset(err)
return nil, err
}
if resp.FastPathVersion != 0 {
req.FastPathVersion = resp.FastPathVersion
bulk.setFastPathVersion(resp.FastPathVersion)
}
req.Dedicated = resp.Dedicated
if resp.AttachToken != "" {
req.AttachToken = resp.AttachToken
}
if req.DataID == 0 {
return nil, errBulkDataIDEmpty
}
bulk := newBulkHandle(c.clientStopContextSnapshot(), runtime, clientFileScope(), req, c.currentClientSessionEpoch(), nil, nil, resp.TransportGeneration, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, c.currentClientSessionEpoch()), clientBulkWriteSender(c, c.currentClientSessionEpoch()), clientBulkReleaseSender(c))
bulk.setClientSnapshotOwner(c)
bulk.markAcceptHandled()
if err := runtime.register(clientFileScope(), bulk); err != nil {
c.releaseBulkDedicatedLane(req.DedicatedLaneID)
_, _ = sendBulkResetClient(context.Background(), c, BulkResetRequest{
BulkID: req.BulkID,
DataID: req.DataID,
Error: err.Error(),
})
if resp.Dedicated {
err = errBulkRejected
c.bestEffortBulkResetAtRoute(route, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: "shared bulk upgraded to dedicated"})
bulk.markReset(err)
return nil, err
}
if bulk.Dedicated() {
if err := c.attachDedicatedBulkSidecar(ctx, bulk); err != nil {
runtime.remove(clientFileScope(), bulk.ID())
_, _ = sendBulkResetClient(context.Background(), c, BulkResetRequest{
BulkID: bulk.ID(),
DataID: bulk.dataIDSnapshot(),
Error: err.Error(),
})
return nil, err
}
if resp.AttachToken != "" {
bulk.setDedicatedAttachToken(resp.AttachToken)
}
bulk.setTransportGeneration(resp.TransportGeneration)
return bulk, nil
}
func (c *ClientCommon) bestEffortBulkReset(req BulkResetRequest) {
if c == nil {
return
}
c.bestEffortBulkResetAtRoute(c.clientSessionRouteSnapshot(), req)
}
func (c *ClientCommon) bestEffortBulkResetAtEpoch(epoch uint64, req BulkResetRequest) {
route := c.clientSessionRouteSnapshot()
route.epoch = epoch
c.bestEffortBulkResetAtRoute(route, req)
}
func (c *ClientCommon) bestEffortBulkResetAtRoute(route clientSessionRoute, req BulkResetRequest) {
if c == nil {
return
}
task := newClientBulkResetRecoveryTaskAtRoute(c, route, req)
q := c.bulkRecoveryQueue()
if !q.enqueue(task) {
c.handleBulkRecoveryOverflowAtRoute(route, req)
}
}
func (c *ClientCommon) bulkRecoveryQueue() *bulkRecoveryQueue {
if c == nil {
return nil
}
c.bulkRecoveryMu.Lock()
defer c.bulkRecoveryMu.Unlock()
if c.bulkRecovery == nil {
c.bulkRecovery = newBulkRecoveryQueue(c.reportBulkRecoveryError)
}
return c.bulkRecovery
}
func newClientBulkResetRecoveryTask(c *ClientCommon, epoch uint64, req BulkResetRequest) bulkRecoveryTask {
route := c.clientSessionRouteSnapshot()
route.epoch = epoch
return newClientBulkResetRecoveryTaskAtRoute(c, route, req)
}
func newClientBulkResetRecoveryTaskAtRoute(c *ClientCommon, route clientSessionRoute, req BulkResetRequest) bulkRecoveryTask {
return func(ctx context.Context) error {
if !c.clientSessionRouteCurrent(route) {
return transportDetachedSessionEpochError()
}
_, err := sendBulkResetClientAtRoute(ctx, c, route, req)
if errors.Is(err, errBulkNotFound) {
return nil
}
return err
}
}
func clientBulkRequest(runtime *bulkRuntime, opt BulkOpenOptions) BulkOpenRequest {
opt = normalizeBulkOpenOptions(opt)
id := opt.ID
@@ -224,8 +318,9 @@ func clientBulkCloseSender(c *ClientCommon) bulkCloseSender {
}
return c.sendDedicatedBulkClose(ctx, bulk, full)
}
_, err := sendBulkCloseClient(ctx, c, BulkCloseRequest{
_, err := sendBulkCloseClientAtRoute(ctx, c, bulk.clientSessionRouteSnapshot(), BulkCloseRequest{
BulkID: bulk.ID(),
DataID: bulk.dataIDSnapshot(),
Full: full,
})
return err
@@ -240,7 +335,7 @@ func clientBulkResetSender(c *ClientCommon) bulkResetSender {
}
return c.sendDedicatedBulkReset(ctx, bulk, message)
}
_, err := sendBulkResetClient(ctx, c, BulkResetRequest{
_, err := sendBulkResetClientAtRoute(ctx, c, bulk.clientSessionRouteSnapshot(), BulkResetRequest{
BulkID: bulk.ID(),
DataID: bulk.dataIDSnapshot(),
Error: message,
@@ -249,7 +344,7 @@ func clientBulkResetSender(c *ClientCommon) bulkResetSender {
}
}
func clientBulkDataSender(c *ClientCommon, epoch uint64) bulkDataSender {
func clientBulkDataSender(c *ClientCommon, route clientSessionRoute) bulkDataSender {
return func(ctx context.Context, bulk *bulkHandle, chunk []byte) error {
if c == nil {
return errBulkClientNil
@@ -267,18 +362,18 @@ func clientBulkDataSender(c *ClientCommon, epoch uint64) bulkDataSender {
}
return c.sendDedicatedBulkData(ctx, bulk, chunk)
}
if epoch != 0 && !c.isClientSessionEpochCurrent(epoch) {
if !c.clientSessionRouteCurrent(route) {
return errTransportDetached
}
dataID := bulk.dataIDSnapshot()
if dataID == 0 {
return errBulkDataPathNotReady
}
return c.sendFastBulkData(ctx, dataID, bulk.nextOutboundDataSeq(), chunk, bulk.fastPathVersionSnapshot())
return c.sendFastBulkDataAtRoute(ctx, route, dataID, bulk.nextOutboundDataSeq(), chunk, bulk.fastPathVersionSnapshot())
}
}
func clientBulkWriteSender(c *ClientCommon, epoch uint64) bulkWriteSender {
func clientBulkWriteSender(c *ClientCommon, route clientSessionRoute) bulkWriteSender {
return func(ctx context.Context, bulk *bulkHandle, startSeq uint64, payload []byte, payloadOwned bool) (int, error) {
if c == nil {
return 0, errBulkClientNil
@@ -296,7 +391,7 @@ func clientBulkWriteSender(c *ClientCommon, epoch uint64) bulkWriteSender {
}
return c.sendDedicatedBulkWrite(ctx, bulk, startSeq, payload, payloadOwned)
}
if epoch != 0 && !c.isClientSessionEpochCurrent(epoch) {
if !c.clientSessionRouteCurrent(route) {
return 0, errTransportDetached
}
if bulk == nil {
@@ -306,11 +401,12 @@ func clientBulkWriteSender(c *ClientCommon, epoch uint64) bulkWriteSender {
if dataID == 0 {
return 0, errBulkDataPathNotReady
}
return c.sendFastBulkWrite(ctx, dataID, startSeq, bulk.chunkSize, bulk.fastPathVersionSnapshot(), payload, payloadOwned)
return c.sendFastBulkWriteAtRoute(ctx, route, dataID, startSeq, bulk.chunkSize, bulk.fastPathVersionSnapshot(), payload, payloadOwned)
}
}
func clientBulkReleaseSender(c *ClientCommon) bulkReleaseSender {
fallbackRoute := c.clientSessionRouteSnapshot()
return func(bulk *bulkHandle, bytes int64, chunks int) error {
if c == nil || bulk == nil {
return errBulkClientNil
@@ -326,14 +422,18 @@ func clientBulkReleaseSender(c *ClientCommon) bulkReleaseSender {
if bulk.Dedicated() {
return c.sendDedicatedBulkRelease(ctx, bulk, bytes, chunks)
}
route := bulk.clientSessionRouteSnapshot()
if !route.bound() {
route = fallbackRoute
}
if bulk.fastPathVersionSnapshot() >= bulkFastPathVersionV2 {
payload, err := encodeBulkDedicatedReleasePayload(bytes, chunks)
if err != nil {
return err
}
return c.sendFastBulkControl(ctx, bulkFastPayloadTypeRelease, 0, bulk.dataIDSnapshot(), 0, bulk.fastPathVersionSnapshot(), payload)
return c.sendFastBulkControlAtRoute(ctx, route, bulkFastPayloadTypeRelease, 0, bulk.dataIDSnapshot(), 0, bulk.fastPathVersionSnapshot(), payload)
}
return sendBulkReleaseClient(ctx, c, BulkReleaseRequest{
return sendBulkReleaseClientAtRoute(ctx, c, route, BulkReleaseRequest{
BulkID: bulk.ID(),
DataID: bulk.dataIDSnapshot(),
Bytes: bytes,