b612/notify
1
0
Fork 0
Code Issues Pull Requests Projects Releases 12 Wiki Activity

fix(notify): 修复传输生命周期竞态，完善背压与协议边界

Browse Source 
- 完善 stream/bulk DataID 分配、预留和双向命名空间，修复并发打开及 dedicated/shared 回退时的 ID 冲突
- 将收发、回复、恢复任务和 sidecar 绑定原始会话与物理连接，防止重连后的旧消息误操作新连接
- 加强 close/reset 身份校验及实例移除检查，修复 dedicated attach 失败、通道引用和资源回收竞态
- 收紧批量发送器停止准入，确保在途入队完成后统一清理请求、缓冲区和等待者
- 修复 record 满队列死锁、取消时序号消耗及关闭竞态，确保关闭有界并返回真实错误
- 增加协商式 record 逻辑半关闭，保留反向 ACK；通过 reset 传递 RecordFailure，避免背压掩盖原始失败原因
- 补齐帧长度、批次数量、序号溢出和未确认窗口校验，提前拒绝超限数据并按字节预算拆批
- 为入站分发增加全局及单连接的条数、字节预算和阻塞背压，关闭时唤醒等待者，消除正常断连日志噪音
- 完善 bulk 窗口释放失败处理与传输诊断，补充并发、重连、背压、协议边界及真实 TCP 回归覆盖
This commit is contained in:
兔子
2026-09-23 15:33:17 +08:00
parent 0826e17063
commit 1f2e74acca
79 changed files with 9190 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bulk_control.go
+167 -43
View File
@@ -3,6 +3,7 @@ package notify
import (
"context"
"errors"
"strings"
"time"
)
@@ -35,6 +36,7 @@ type BulkOpenResponse struct {
type BulkCloseRequest struct {
BulkID string
DataID uint64
Full bool
}
@@ -157,6 +159,14 @@ func dispatchBulkAccept(handler func(BulkAcceptInfo) error, bulk *bulkHandle, in
if bulk == nil {
return errBulkNotFound
}
if !bulk.acceptDispatchAllowed() {
if resetErr := bulk.resetErrSnapshot(); resetErr != nil {
return resetErr
}
err := transportDetachedErrorForTransport(bulk.TransportConn())
bulk.markReset(err)
return err
}
if !bulk.markAcceptDispatched() {
return nil
}
@@ -178,6 +188,7 @@ func dispatchBulkAccept(handler func(BulkAcceptInfo) error, bulk *bulkHandle, in
}
func (c *ClientCommon) clientBulkAcceptReadyNotifier(bulk *bulkHandle) func(error) {
route := bulk.clientSessionRouteSnapshot()
return func(readyErr error) {
if c == nil || bulk == nil {
return
@@ -191,7 +202,7 @@ func (c *ClientCommon) clientBulkAcceptReadyNotifier(bulk *bulkHandle) func(erro
}
ctx, cancel := context.WithTimeout(context.Background(), defaultBulkAcceptReadyTimeout)
defer cancel()
if _, err := sendBulkReadyClient(ctx, c, req); err != nil && bulk.Context().Err() == nil {
if _, err := sendBulkReadyClientAtRoute(ctx, c, route, req); err != nil && bulk.Context().Err() == nil {
bulk.markReset(err)
}
}
@@ -202,11 +213,8 @@ func sendBulkReadyServer(ctx context.Context, s *ServerCommon, logical *LogicalC
return errBulkServerNil
}
if transport != nil {
if _, err := sendBulkReadyServerTransport(ctx, s, transport, req); err == nil {
return nil
} else if !errors.Is(err, errTransportDetached) && !errors.Is(err, errBulkTransportNil) {
return err
}
_, err := sendBulkReadyServerTransport(ctx, s, transport, req)
return err
}
if logical == nil {
return errBulkLogicalConnNil
@@ -296,6 +304,15 @@ func (c *ClientCommon) handleInboundBulkOpen(msg *Message) {
replyBulkControlIfNeeded(msg, resp)
return
}
route := msg.clientRoute
if !route.bound() {
route = c.clientSessionRouteSnapshot()
}
if err := c.ensureClientSessionRouteSendReady(route); err != nil {
resp.Error = err.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if req.Dedicated {
if err := clientDedicatedBulkSupportError(c); err != nil {
resp.Error = err.Error()
@@ -310,17 +327,38 @@ func (c *ClientCommon) handleInboundBulkOpen(msg *Message) {
return
}
scope := clientFileScope()
if req.DataID == 0 {
req.DataID = runtime.nextDataID()
resp.DataID = req.DataID
if existing, ok := runtime.lookup(scope, req.BulkID); ok && !existing.acceptsClientSessionRoute(route) {
existing.markReset(transportDetachedSessionEpochError())
}
if req.Dedicated && req.AttachToken == "" {
req.AttachToken = newBulkAttachToken()
}
resp.AttachToken = req.AttachToken
bulk := newBulkHandle(c.clientStopContextSnapshot(), runtime, scope, req, c.currentClientSessionEpoch(), nil, nil, 0, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, c.currentClientSessionEpoch()), clientBulkWriteSender(c, c.currentClientSessionEpoch()), clientBulkReleaseSender(c))
bulk := newBulkHandle(clientSessionRouteContext(route), runtime, scope, req, route.epoch, nil, nil, 0, clientBulkCloseSender(c), clientBulkResetSender(c), clientBulkDataSender(c, route), clientBulkWriteSender(c, route), clientBulkReleaseSender(c))
bulk.setClientSnapshotOwner(c)
if err := runtime.register(scope, bulk); err != nil {
bulk.setClientSessionRoute(route)
if req.Dedicated {
if err := c.retainBulkDedicatedLaneAtRoute(bulk.dedicatedLaneIDSnapshot(), route); err != nil {
// newBulkHandle starts its write/release workers before the lane
// retain can fail. Reset the unadopted candidate so those workers and
// its context are reclaimed; no lane lease was acquired yet.
bulk.markReset(err)
resp.Error = err.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
bulk.markDedicatedLaneReserved()
}
if err := runtime.adoptInbound(scope, bulk); err != nil {
resp.Error = err.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if err := c.ensureClientSessionRouteSendReady(route); err != nil {
// Reattach may have won the race after the preflight check. Remove the
// just-registered handle before any handler or sidecar side effect.
runtime.remove(scope, bulk)
bulk.markReset(err)
resp.Error = err.Error()
replyBulkControlIfNeeded(msg, resp)
return
@@ -376,6 +414,11 @@ func (s *ServerCommon) handleInboundBulkOpen(msg *Message) {
return
}
transport := messageTransportConnSnapshot(msg)
if transport != nil && !transport.IsCurrent() {
resp.Error = transportDetachedErrorForTransport(transport).Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if req.Dedicated {
if err := logicalDedicatedBulkSupportError(logical); err != nil {
resp.Error = err.Error()
@@ -391,20 +434,28 @@ func (s *ServerCommon) handleInboundBulkOpen(msg *Message) {
}
}
scope := serverFileScope(logical)
if req.DataID == 0 {
req.DataID = runtime.nextDataID()
resp.DataID = req.DataID
if existing, ok := runtime.lookup(scope, req.BulkID); ok && !existing.acceptsTransportGeneration(transport) {
existing.markReset(transportDetachedGenerationMismatchError(existing.TransportGeneration(), transport))
}
if req.Dedicated && req.AttachToken == "" {
req.AttachToken = newBulkAttachToken()
}
resp.AttachToken = req.AttachToken
bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, bulkTransportGeneration(logical, transport), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport))
if err := runtime.register(scope, bulk); err != nil {
if err := runtime.adoptInbound(scope, bulk); err != nil {
resp.Error = err.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if transport != nil && !transport.IsCurrent() {
// Reattach may have won the race after the preflight check. Remove the
// just-registered handle before any handler or sidecar side effect.
runtime.remove(scope, bulk)
bulk.markReset(transportDetachedErrorForTransport(transport))
resp.Error = transportDetachedErrorForTransport(transport).Error()
replyBulkControlIfNeeded(msg, resp)
return
}
s.attachServerDedicatedSidecarIfExists(logical, bulk)
if runtime.handlerSnapshot() == nil {
bulk.markReset(errBulkHandlerNotConfigured)
@@ -447,12 +498,17 @@ func (c *ClientCommon) handleInboundBulkClose(msg *Message) {
replyBulkControlIfNeeded(msg, resp)
return
}
bulk, ok := runtime.lookup(clientFileScope(), req.BulkID)
bulk, ok := runtime.lookupControl(clientFileScope(), req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if !bulk.acceptsClientSessionRoute(msg.clientRoute) {
resp.Error = transportDetachedSessionEpochError().Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if req.Full {
bulk.markPeerClosed()
} else {
@@ -478,12 +534,17 @@ func (s *ServerCommon) handleInboundBulkClose(msg *Message) {
}
logical := messageLogicalConnSnapshot(msg)
scope := serverFileScope(logical)
bulk, ok := runtime.lookup(scope, req.BulkID)
bulk, ok := runtime.lookupControl(scope, req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if !bulk.acceptsTransportGeneration(messageTransportConnSnapshot(msg)) {
resp.Error = transportDetachedGenerationMismatchError(bulk.TransportGeneration(), messageTransportConnSnapshot(msg)).Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if req.Full {
bulk.markPeerClosed()
} else {
@@ -507,15 +568,17 @@ func (c *ClientCommon) handleInboundBulkReset(msg *Message) {
replyBulkControlIfNeeded(msg, resp)
return
}
bulk, ok := runtime.lookup(clientFileScope(), req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(clientFileScope(), req.DataID)
}
bulk, ok := runtime.lookupControl(clientFileScope(), req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if !bulk.acceptsClientSessionRoute(msg.clientRoute) {
resp.Error = transportDetachedSessionEpochError().Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if resp.BulkID == "" {
resp.BulkID = bulk.ID()
}
@@ -533,13 +596,13 @@ func (c *ClientCommon) handleInboundBulkRelease(msg *Message) {
if runtime == nil {
return
}
bulk, ok := runtime.lookup(clientFileScope(), req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(clientFileScope(), req.DataID)
}
bulk, ok := runtime.lookupControl(clientFileScope(), req.BulkID, req.DataID)
if !ok {
return
}
if !bulk.acceptsClientSessionRoute(msg.clientRoute) {
return
}
bulk.releaseOutboundWindow(req.Bytes, req.Chunks)
}
@@ -557,15 +620,17 @@ func (c *ClientCommon) handleInboundBulkReady(msg *Message) {
replyBulkControlIfNeeded(msg, resp)
return
}
bulk, ok := runtime.lookup(clientFileScope(), req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(clientFileScope(), req.DataID)
}
bulk, ok := runtime.lookupControl(clientFileScope(), req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if !bulk.acceptsClientSessionRoute(msg.clientRoute) {
resp.Error = transportDetachedSessionEpochError().Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if resp.BulkID == "" {
resp.BulkID = bulk.ID()
}
@@ -594,15 +659,18 @@ func (s *ServerCommon) handleInboundBulkReset(msg *Message) {
}
logical := messageLogicalConnSnapshot(msg)
scope := serverFileScope(logical)
bulk, ok := runtime.lookup(scope, req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(scope, req.DataID)
}
bulk, ok := runtime.lookupControl(scope, req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
transport := messageTransportConnSnapshot(msg)
if !bulk.acceptsTransportGeneration(transport) {
resp.Error = transportDetachedGenerationMismatchError(bulk.TransportGeneration(), transport).Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if resp.BulkID == "" {
resp.BulkID = bulk.ID()
}
@@ -627,15 +695,18 @@ func (s *ServerCommon) handleInboundBulkReady(msg *Message) {
}
logical := messageLogicalConnSnapshot(msg)
scope := serverFileScope(logical)
bulk, ok := runtime.lookup(scope, req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(scope, req.DataID)
}
bulk, ok := runtime.lookupControl(scope, req.BulkID, req.DataID)
if !ok {
resp.Error = errBulkNotFound.Error()
replyBulkControlIfNeeded(msg, resp)
return
}
transport := messageTransportConnSnapshot(msg)
if !bulk.acceptsTransportGeneration(transport) {
resp.Error = transportDetachedGenerationMismatchError(bulk.TransportGeneration(), transport).Error()
replyBulkControlIfNeeded(msg, resp)
return
}
if resp.BulkID == "" {
resp.BulkID = bulk.ID()
}
@@ -659,13 +730,13 @@ func (s *ServerCommon) handleInboundBulkRelease(msg *Message) {
}
logical := messageLogicalConnSnapshot(msg)
scope := serverFileScope(logical)
bulk, ok := runtime.lookup(scope, req.BulkID)
if !ok && req.DataID != 0 {
bulk, ok = runtime.lookupByDataID(scope, req.DataID)
}
bulk, ok := runtime.lookupControl(scope, req.BulkID, req.DataID)
if !ok {
return
}
if !bulk.acceptsTransportGeneration(messageTransportConnSnapshot(msg)) {
return
}
bulk.releaseOutboundWindow(req.Bytes, req.Chunks)
}
@@ -687,6 +758,17 @@ func sendBulkOpenClient(ctx context.Context, c Client, req BulkOpenRequest) (Bul
return decodeBulkOpenResponse(msg)
}
func sendBulkOpenClientAtRoute(ctx context.Context, c *ClientCommon, route clientSessionRoute, req BulkOpenRequest) (BulkOpenResponse, error) {
if c == nil {
return BulkOpenResponse{}, errBulkClientNil
}
msg, err := c.sendObjCtxAtRoute(ctx, route, BulkOpenSignalKey, req)
if err != nil {
return BulkOpenResponse{}, err
}
return decodeBulkOpenResponse(msg)
}
func sendBulkOpenServerLogical(ctx context.Context, s Server, logical *LogicalConn, req BulkOpenRequest) (BulkOpenResponse, error) {
if s == nil {
return BulkOpenResponse{}, errBulkServerNil
@@ -726,6 +808,17 @@ func sendBulkCloseClient(ctx context.Context, c Client, req BulkCloseRequest) (B
return decodeBulkCloseResponse(msg)
}
func sendBulkCloseClientAtRoute(ctx context.Context, c *ClientCommon, route clientSessionRoute, req BulkCloseRequest) (BulkCloseResponse, error) {
if c == nil {
return BulkCloseResponse{}, errBulkClientNil
}
msg, err := c.sendObjCtxAtRoute(ctx, route, BulkCloseSignalKey, req)
if err != nil {
return BulkCloseResponse{}, err
}
return decodeBulkCloseResponse(msg)
}
func sendBulkCloseServerLogical(ctx context.Context, s Server, logical *LogicalConn, req BulkCloseRequest) (BulkCloseResponse, error) {
if s == nil {
return BulkCloseResponse{}, errBulkServerNil
@@ -765,6 +858,17 @@ func sendBulkResetClient(ctx context.Context, c Client, req BulkResetRequest) (B
return decodeBulkResetResponse(msg)
}
func sendBulkResetClientAtRoute(ctx context.Context, c *ClientCommon, route clientSessionRoute, req BulkResetRequest) (BulkResetResponse, error) {
if c == nil {
return BulkResetResponse{}, errBulkClientNil
}
msg, err := c.sendObjCtxAtRoute(ctx, route, BulkResetSignalKey, req)
if err != nil {
return BulkResetResponse{}, err
}
return decodeBulkResetResponse(msg)
}
func sendBulkResetServerLogical(ctx context.Context, s Server, logical *LogicalConn, req BulkResetRequest) (BulkResetResponse, error) {
if s == nil {
return BulkResetResponse{}, errBulkServerNil
@@ -794,6 +898,10 @@ func sendBulkResetServerTransport(ctx context.Context, s Server, transport *Tran
}
func sendBulkReleaseClient(ctx context.Context, c *ClientCommon, req BulkReleaseRequest) error {
return sendBulkReleaseClientAtRoute(ctx, c, c.clientSessionRouteSnapshot(), req)
}
func sendBulkReleaseClientAtRoute(ctx context.Context, c *ClientCommon, route clientSessionRoute, req BulkReleaseRequest) error {
if c == nil {
return errBulkClientNil
}
@@ -801,11 +909,11 @@ func sendBulkReleaseClient(ctx context.Context, c *ClientCommon, req BulkRelease
if err != nil {
return err
}
_, err = c.sendWithContext(ctx, TransferMsg{
_, err = c.sendWithContextTimeoutAtRoute(ctx, route, TransferMsg{
Key: BulkReleaseSignalKey,
Value: data,
Type: MSG_ASYNC,
})
}, 0)
return err
}
@@ -974,6 +1082,9 @@ func bulkControlResultError(op string, accepted bool, message string, callErr er
}
func bulkControlMessageError(message string) error {
if message == errTransportDetached.Error() || strings.HasPrefix(message, errTransportDetached.Error()+":") {
return errTransportDetached
}
switch message {
case errBulkNotFound.Error():
return errBulkNotFound
@@ -993,6 +1104,8 @@ func bulkControlMessageError(message string) error {
return errBulkRangeInvalid
case errBulkDataIDEmpty.Error():
return errBulkDataIDEmpty
case errBulkDataIDExhausted.Error():
return errBulkDataIDExhausted
default:
return errors.New(message)
}
@@ -1023,6 +1136,17 @@ func sendBulkReadyClient(ctx context.Context, c Client, req BulkReadyRequest) (B
return decodeBulkReadyResponse(msg)
}
func sendBulkReadyClientAtRoute(ctx context.Context, c *ClientCommon, route clientSessionRoute, req BulkReadyRequest) (BulkReadyResponse, error) {
if c == nil {
return BulkReadyResponse{}, errBulkClientNil
}
msg, err := c.sendObjCtxAtRoute(ctx, route, BulkReadySignalKey, req)
if err != nil {
return BulkReadyResponse{}, err
}
return decodeBulkReadyResponse(msg)
}
func sendBulkReadyServerLogical(ctx context.Context, s Server, logical *LogicalConn, req BulkReadyRequest) (BulkReadyResponse, error) {
if s == nil {
return BulkReadyResponse{}, errBulkServerNil