fix(notify): 根治低带宽控制面阻塞与传输写入卡死

- 为控制消息增加优先级、公平调度、队列字节预算和自适应批处理 - 支持可取消的写门等待，收紧 shared/dedicated bulk、stream 和 Reply 写入边界 - 修复 bulk reset/close、连接 handoff 和安全 profile 切换时序 - 保留旧取消与超时错误契约，新增阶段化 TransportSendError - 增加 ReplyCtx、ReplyObjCtx、写超时配置及黑洞连接和竞态回归测试
2026-08-14 10:16:37 +08:00
parent 98ef9e7fcc
commit 0826e17063
45 changed files with 4231 additions and 293 deletions
@@ -34,20 +34,35 @@ func (s *ServerCommon) send(c *ClientConn, msg TransferMsg) (WaitMsg, error) {
 }

 func (s *ServerCommon) sendLogical(logical *LogicalConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendLogicalContext(context.Background(), logical, msg, 0)
+}
+
+func (s *ServerCommon) sendLogicalContext(ctx context.Context, logical *LogicalConn, msg TransferMsg, writeTimeout time.Duration) (WaitMsg, error) {
 	if logical == nil {
-		return s.sendTransport(nil, msg)
+		return s.sendTransportContextWithWriteTimeout(ctx, nil, msg, writeTimeout)
 	}
-	return s.sendTransport(s.resolveOutboundTransport(logical), msg)
+	return s.sendTransportContextWithWriteTimeout(ctx, s.resolveOutboundTransport(logical), msg, writeTimeout)
 }

 func (s *ServerCommon) sendTransport(transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendTransportContext(context.Background(), transport, msg)
+}
+
+func (s *ServerCommon) sendTransportContext(ctx context.Context, transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendTransportContextWithWriteTimeout(ctx, transport, msg, 0)
+}
+
+func (s *ServerCommon) sendTransportContextWithWriteTimeout(ctx context.Context, transport *TransportConn, msg TransferMsg, writeTimeout time.Duration) (WaitMsg, error) {
 	if err := s.ensureServerTransportSendReady(transport); err != nil {
 		return WaitMsg{}, err
 	}
-	if s.serverUDPListenerSnapshot() != nil {
-		return s.sendUDPTransport(transport, msg)
+	if ctx == nil {
+		ctx = context.Background()
 	}
-	return s.sendTUTransport(transport, msg)
+	if s.serverUDPListenerSnapshot() != nil {
+		return s.sendUDPTransportContextWithWriteTimeout(ctx, transport, msg, writeTimeout)
+	}
+	return s.sendTUTransportContextWithWriteTimeout(ctx, transport, msg, writeTimeout)
 }

 func (s *ServerCommon) sendTU(c *ClientConn, msg TransferMsg) (WaitMsg, error) {
@@ -62,6 +77,23 @@ func (s *ServerCommon) sendTULogical(logical *LogicalConn, msg TransferMsg) (Wai
 }

 func (s *ServerCommon) sendTUTransport(transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendTUTransportContext(context.Background(), transport, msg)
+}
+
+func (s *ServerCommon) sendTUTransportContext(ctx context.Context, transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendTUTransportContextWithWriteTimeout(ctx, transport, msg, 0)
+}
+
+func (s *ServerCommon) sendTUTransportContextWithWriteTimeout(ctx context.Context, transport *TransportConn, msg TransferMsg, writeTimeout time.Duration) (WaitMsg, error) {
+	if err := s.ensureServerTransportSendReady(transport); err != nil {
+		return WaitMsg{}, err
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return WaitMsg{}, err
+	}
 	var wait WaitMsg
 	if msg.Type != MSG_SYNC_REPLY && msg.Type != MSG_KEY_CHANGE && msg.Type != MSG_SYS_REPLY || msg.ID == 0 {
 		msg.ID = atomic.AddUint64(&s.msgID, 1)
@@ -74,6 +106,8 @@ func (s *ServerCommon) sendTUTransport(transport *TransportConn, msg TransferMsg
 	if err != nil {
 		return WaitMsg{}, err
 	}
+	env.controlCtx = ctx
+	env.controlTimeout = writeTimeout
 	if requiresSignalReplyWait(msg) {
 		wait = s.getPendingWaitPool().createAndStoreWithScope(msg, serverTransportScopeForTransport(transport))
 	}
@@ -121,12 +155,18 @@ func (s *ServerCommon) sendWaitLogical(logical *LogicalConn, msg TransferMsg, ti
 }

 func (s *ServerCommon) sendTransportWait(transport *TransportConn, msg TransferMsg, timeout time.Duration) (Message, error) {
-	data, err := s.sendTransport(transport, msg)
+	ctx := context.Background()
+	cancel := func() {}
+	if timeout != 0 {
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+	}
+	defer cancel()
+	data, err := s.sendTransportContext(ctx, transport, msg)
 	if err != nil {
-		return Message{}, err
+		return Message{}, publicContextSendError(ctx, err)
 	}
 	stopCh := sessionStopChan(s.serverStopContextSnapshot())
-	if timeout.Seconds() == 0 {
+	if timeout == 0 {
 		msg, ok := <-data.Reply
 		if !ok {
 			return msg, pendingWaitClosedErrorWith(stopCh, transportDetachedErrorForTransport(transport))
@@ -134,7 +174,7 @@ func (s *ServerCommon) sendTransportWait(transport *TransportConn, msg TransferM
 		return msg, nil
 	}
 	select {
-	case <-time.After(timeout):
+	case <-ctx.Done():
 		s.getPendingWaitPool().removeAndClose(data.TransferMsg.ID)
 		return Message{}, os.ErrDeadlineExceeded
 	case <-stopCh:
@@ -191,14 +231,14 @@ func (s *ServerCommon) SendCtxTransport(ctx context.Context, t *TransportConn, k
 }

 func (s *ServerCommon) sendCtxTransport(t *TransportConn, msg TransferMsg, ctx context.Context) (Message, error) {
-	data, err := s.sendTransport(t, msg)
-	if err != nil {
-		return Message{}, err
-	}
-	stopCh := sessionStopChan(s.serverStopContextSnapshot())
 	if ctx == nil {
 		ctx = context.Background()
 	}
+	data, err := s.sendTransportContext(ctx, t, msg)
+	if err != nil {
+		return Message{}, publicContextSendError(ctx, err)
+	}
+	stopCh := sessionStopChan(s.serverStopContextSnapshot())
 	select {
 	case <-ctx.Done():
 		s.getPendingWaitPool().removeAndClose(data.TransferMsg.ID)
@@ -307,6 +347,23 @@ func (s *ServerCommon) sendUDPLogical(logical *LogicalConn, msg TransferMsg) (Wa
 }

 func (s *ServerCommon) sendUDPTransport(transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendUDPTransportContext(context.Background(), transport, msg)
+}
+
+func (s *ServerCommon) sendUDPTransportContext(ctx context.Context, transport *TransportConn, msg TransferMsg) (WaitMsg, error) {
+	return s.sendUDPTransportContextWithWriteTimeout(ctx, transport, msg, 0)
+}
+
+func (s *ServerCommon) sendUDPTransportContextWithWriteTimeout(ctx context.Context, transport *TransportConn, msg TransferMsg, writeTimeout time.Duration) (WaitMsg, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := s.ensureServerTransportSendReady(transport); err != nil {
+		return WaitMsg{}, err
+	}
+	if err := ctx.Err(); err != nil {
+		return WaitMsg{}, err
+	}
 	var wait WaitMsg
 	if msg.Type != MSG_SYNC_REPLY && msg.Type != MSG_KEY_CHANGE && msg.Type != MSG_SYS_REPLY || msg.ID == 0 {
 		msg.ID = uint64(time.Now().UnixNano()) + rand.Uint64() + rand.Uint64()
@@ -315,6 +372,8 @@ func (s *ServerCommon) sendUDPTransport(transport *TransportConn, msg TransferMs
 	if err != nil {
 		return WaitMsg{}, err
 	}
+	env.controlCtx = ctx
+	env.controlTimeout = writeTimeout
 	if requiresSignalReplyWait(msg) {
 		wait = s.getPendingWaitPool().createAndStoreWithScope(msg, serverTransportScopeForTransport(transport))
 	}
@@ -347,14 +406,20 @@ func (s *ServerCommon) sendEnvelopeTransport(transport *TransportConn, env Envel
 	if logical == nil {
 		return transportDetachedErrorForTransport(transport)
 	}
-	payload, err := s.encodeEnvelopePayloadLogical(logical, env)
+	var payload []byte
+	var err error
+	if env.transportProfile != nil {
+		payload, err = s.encodeEnvelopePayloadInbound(logical, env, env.transportProfile)
+	} else {
+		payload, err = s.encodeEnvelopePayloadLogical(logical, env)
+	}
 	if err != nil {
 		return err
 	}
 	if batchedControlEnvelope(env) {
-		return s.writeControlEnvelopePayload(logical, transport, nil, payload)
+		return s.writeControlEnvelopePayload(logical, transport, nil, env.controlContext(), payload, env.controlPriority, env.controlTimeout)
 	}
-	return s.writeEnvelopePayload(logical, transport, nil, payload)
+	return s.writeEnvelopePayloadContextTimeout(env.controlContext(), logical, transport, nil, payload, env.controlTimeout)
 }

 func (s *ServerCommon) sendEnvelopeInboundTransport(logical *LogicalConn, transport *TransportConn, conn net.Conn, env Envelope) error {
@@ -376,34 +441,34 @@ func (s *ServerCommon) sendEnvelopeInboundTransportWithProfile(logical *LogicalC
 		return err
 	}
 	if batchedControlEnvelope(env) {
-		return s.writeControlEnvelopePayload(logical, transport, conn, payload)
+		return s.writeControlEnvelopePayload(logical, transport, conn, env.controlContext(), payload, env.controlPriority, env.controlTimeout)
 	}
-	return s.writeEnvelopePayload(logical, transport, conn, payload)
+	return s.writeEnvelopePayloadContextTimeout(env.controlContext(), logical, transport, conn, payload, env.controlTimeout)
 }

-func (s *ServerCommon) writeControlEnvelopePayload(logical *LogicalConn, transport *TransportConn, conn net.Conn, payload []byte) error {
+func (s *ServerCommon) writeControlEnvelopePayload(logical *LogicalConn, transport *TransportConn, conn net.Conn, ctx context.Context, payload []byte, priority controlPriority, writeTimeout time.Duration) error {
 	if logical == nil {
 		return transportDetachedErrorForPeer(logical, transport)
 	}
 	if s.serverUDPListenerSnapshot() != nil {
-		return s.writeEnvelopePayload(logical, transport, conn, payload)
+		return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, writeTimeout)
 	}
 	binding := logical.transportBindingSnapshot()
 	if binding == nil || binding.queueSnapshot() == nil {
-		return s.writeEnvelopePayload(logical, transport, conn, payload)
+		return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, writeTimeout)
 	}
 	boundConn := binding.connSnapshot()
 	if boundConn == nil || isPacketTransportConn(boundConn) {
-		return s.writeEnvelopePayload(logical, transport, conn, payload)
+		return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, writeTimeout)
 	}
 	if conn != nil && conn != boundConn {
-		return s.writeEnvelopePayload(logical, transport, conn, payload)
+		return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, writeTimeout)
 	}
 	sender := binding.controlBatchSenderSnapshot()
 	if sender == nil {
-		return s.writeEnvelopePayload(logical, transport, conn, payload)
+		return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, writeTimeout)
 	}
-	return sender.submit(payload, writeDeadlineFromTimeout(logical.maxWriteTimeoutSnapshot()))
+	return sender.submitContext(ctx, payload, shorterPositiveDuration(logical.maxWriteTimeoutSnapshot(), writeTimeout), priority)
 }

 func (s *ServerCommon) encodeEnvelopePayloadInbound(logical *LogicalConn, env Envelope, profile *transportProtectionProfile) ([]byte, error) {
@@ -418,6 +483,10 @@ func (s *ServerCommon) encodeEnvelopePayloadInbound(logical *LogicalConn, env En
 }

 func (s *ServerCommon) sendTransferInbound(logical *LogicalConn, transport *TransportConn, conn net.Conn, profile *transportProtectionProfile, msg TransferMsg) error {
+	return s.sendTransferInboundContext(context.Background(), logical, transport, conn, profile, msg, 0)
+}
+
+func (s *ServerCommon) sendTransferInboundContext(ctx context.Context, logical *LogicalConn, transport *TransportConn, conn net.Conn, profile *transportProtectionProfile, msg TransferMsg, writeTimeout time.Duration) error {
 	if logical == nil && transport != nil {
 		logical = transport.logicalConnSnapshot()
 	}
@@ -428,10 +497,30 @@ func (s *ServerCommon) sendTransferInbound(logical *LogicalConn, transport *Tran
 	if err != nil {
 		return err
 	}
+	env.controlCtx = ctx
+	env.controlTimeout = writeTimeout
 	return s.sendEnvelopeInboundTransportWithProfile(logical, transport, conn, profile, env)
 }

 func (s *ServerCommon) writeEnvelopePayload(logical *LogicalConn, transport *TransportConn, conn net.Conn, payload []byte) error {
+	return s.writeEnvelopePayloadContext(context.Background(), logical, transport, conn, payload)
+}
+
+func (s *ServerCommon) writeEnvelopePayloadContext(ctx context.Context, logical *LogicalConn, transport *TransportConn, conn net.Conn, payload []byte) error {
+	return s.writeEnvelopePayloadContextTimeout(ctx, logical, transport, conn, payload, 0)
+}
+
+func (s *ServerCommon) writeEnvelopePayloadContextTimeout(ctx context.Context, logical *LogicalConn, transport *TransportConn, conn net.Conn, payload []byte, writeTimeout time.Duration) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if logical == nil {
+		return transportDetachedErrorForPeer(logical, transport)
+	}
+	writeTimeout = shorterPositiveDuration(logical.maxWriteTimeoutSnapshot(), writeTimeout)
 	udpListener := s.serverUDPListenerSnapshot()
 	queue := s.serverQueueSnapshot()
 	if queue == nil {
@@ -441,12 +530,18 @@ func (s *ServerCommon) writeEnvelopePayload(logical *LogicalConn, transport *Tra
 		if transport == nil || transport.RemoteAddr() == nil {
 			return transportDetachedErrorForTransport(transport)
 		}
-		if timeout := logical.maxWriteTimeoutSnapshot(); timeout > 0 {
-			_ = udpListener.SetWriteDeadline(time.Now().Add(timeout))
-		}
 		data := queue.BuildMessage(payload)
-		_, err := udpListener.WriteTo(data, transport.RemoteAddr())
-		return err
+		deadline := earlierWriteDeadline(writeDeadlineFromTimeout(writeTimeout), contextDeadline(ctx))
+		return s.withUDPWriteLockDeadline(ctx, deadline, func() error {
+			if !deadline.IsZero() {
+				if err := udpListener.SetWriteDeadline(deadline); err != nil {
+					return err
+				}
+				defer func() { _ = udpListener.SetWriteDeadline(time.Time{}) }()
+			}
+			_, err := udpListener.WriteTo(data, transport.RemoteAddr())
+			return err
+		})
 	}
 	var binding *transportBinding
 	if logical != nil {
@@ -456,33 +551,58 @@ func (s *ServerCommon) writeEnvelopePayload(logical *LogicalConn, transport *Tra
 		if binding == nil {
 			return os.ErrClosed
 		}
-		return binding.withConnWriteLock(func(conn net.Conn) error {
-			if timeout := logical.maxWriteTimeoutSnapshot(); timeout > 0 {
-				if err := conn.SetWriteDeadline(time.Now().Add(timeout)); err != nil {
-					return err
-				}
-			}
+		lockAcquired, err := binding.withConnWriteLockContextStopTimeout(ctx, nil, writeTimeout, func(conn net.Conn) error {
 			return writeFramedPayloadUnlocked(conn, queue, payload)
 		})
+		if lockAcquired && err != nil {
+			binding.closeConn()
+		}
+		return err
 	}
 	if binding != nil && binding.connSnapshot() == conn {
-		return binding.withConnWriteLock(func(conn net.Conn) error {
-			if timeout := logical.maxWriteTimeoutSnapshot(); timeout > 0 {
-				if err := conn.SetWriteDeadline(time.Now().Add(timeout)); err != nil {
-					return err
-				}
-			}
+		lockAcquired, err := binding.withConnWriteLockContextStopTimeout(ctx, nil, writeTimeout, func(conn net.Conn) error {
 			return writeFramedPayloadUnlocked(conn, queue, payload)
 		})
-	}
-	return withRawConnWriteLock(conn, func(conn net.Conn) error {
-		if timeout := logical.maxWriteTimeoutSnapshot(); timeout > 0 {
-			if err := conn.SetWriteDeadline(time.Now().Add(timeout)); err != nil {
-				return err
-			}
+		if lockAcquired && err != nil {
+			binding.closeConn()
 		}
+		return err
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	deadline := earlierWriteDeadline(writeDeadlineFromTimeout(writeTimeout), contextDeadline(ctx))
+	writeStarted, err := withRawConnWriteLockContextDeadline(ctx, conn, deadline, func(conn net.Conn) error {
 		return writeFramedPayloadUnlocked(conn, queue, payload)
 	})
+	if writeStarted && err != nil && !isPacketTransportConn(conn) {
+		_ = conn.Close()
+	}
+	return err
+}
+
+func (s *ServerCommon) withUDPWriteLock(ctx context.Context, fn func() error) error {
+	return s.withUDPWriteLockDeadline(ctx, contextDeadline(ctx), fn)
+}
+
+func (s *ServerCommon) withUDPWriteLockDeadline(ctx context.Context, deadline time.Time, fn func() error) error {
+	if s == nil {
+		return net.ErrClosed
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	s.udpWriteGateOnce.Do(func() {
+		s.udpWriteGate = newConnWriteGate()
+	})
+	if err := lockWriteGateContextDeadline(ctx, nil, s.udpWriteGate, deadline); err != nil {
+		return err
+	}
+	defer func() { s.udpWriteGate <- struct{}{} }()
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return fn()
 }

 func (s *ServerCommon) dispatchEnvelope(logical *LogicalConn, transport *TransportConn, conn net.Conn, env Envelope, now time.Time) {