fix: 修复 dedicated bulk attach 竞态并优化 short write 补写路径

- 客户端 dedicated attach 回复改为精确读取单帧，避免 attach reply 与后续 NBR1 数据粘连后被误解析 - 服务端 accepted attach 改为先 detach transport，再直接回 attach reply，随后立即切入 dedicated bulk read loop - transport 读循环在 stop 或 transport ownership 失效后不再继续上推已读数据，避免 handoff 后首包被旧 reader 吃掉 - dedicated bulk record 写路径改为 full-write，消除 short write 导致的 invalid bulk fast payload - 优化 vectored write 补写策略：先尝试一次 writev，未写完时直接顺序补完剩余 buffers，减少重复 WriteTo 开销 - 放宽 vectored write 能力识别，支持通过 UnwrapConn/WriteBuffers 命中 fast path - 修复 dedicated batch 排队路径 payload 复用问题，改为深拷贝 queued items - 补齐 dedicated attach、short write、payload clone、transport stop/handoff 等回归测试
2026-04-16 17:27:48 +08:00
parent 7ed3dd5b37
commit 4f760f2807
9 changed files with 690 additions and 47 deletions
@@ -3,11 +3,13 @@ package notify
 import (
 	"b612.me/notify/internal/transport"
 	"b612.me/stario"
+	"bytes"
 	"context"
 	cryptorand "crypto/rand"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"sync/atomic"
@@ -19,8 +21,17 @@ const (
 	bulkDedicatedRecordMagic     = "NBR1"
 	bulkDedicatedRecordHeaderLen = 8
 	bulkDedicatedAttachTimeout   = 5 * time.Second
+
+	bulkDedicatedAttachFrameMagicSize     = 8
+	bulkDedicatedAttachFrameHeaderLen     = 14
+	bulkDedicatedAttachFrameVersionOffset = 12
+	bulkDedicatedAttachFrameFlagsOffset   = 13
+	bulkDedicatedAttachFrameVersionV1     = 1
+	bulkDedicatedAttachFrameFlagsNone     = 0
 )

+var bulkDedicatedAttachFrameMagic = [bulkDedicatedAttachFrameMagicSize]byte{11, 27, 19, 96, 12, 25, 2, 20}
+
 type bulkAttachRequest struct {
 	PeerID      string
 	BulkID      string
@@ -121,6 +132,35 @@ func decodeDirectSignalPayload(sequenceDe func([]byte) (interface{}, error), msg
 	return unwrapTransferMsgEnvelope(env, sequenceDe)
 }

+func readDirectSignalFramePayload(conn net.Conn) ([]byte, error) {
+	if conn == nil {
+		return nil, net.ErrClosed
+	}
+	var header [bulkDedicatedAttachFrameHeaderLen]byte
+	if _, err := io.ReadFull(conn, header[:]); err != nil {
+		return nil, err
+	}
+	if !bytes.Equal(header[:bulkDedicatedAttachFrameMagicSize], bulkDedicatedAttachFrameMagic[:]) {
+		return nil, stario.ErrQueueDataFormat
+	}
+	if got := header[bulkDedicatedAttachFrameVersionOffset]; got != bulkDedicatedAttachFrameVersionV1 {
+		return nil, stario.ErrQueueUnsupportedVersion
+	}
+	if got := header[bulkDedicatedAttachFrameFlagsOffset]; got != bulkDedicatedAttachFrameFlagsNone {
+		return nil, stario.ErrQueueUnsupportedFlags
+	}
+	length := binary.BigEndian.Uint32(header[bulkDedicatedAttachFrameMagicSize : bulkDedicatedAttachFrameMagicSize+4])
+	maxInt := int(^uint(0) >> 1)
+	if uint64(length) > uint64(maxInt) {
+		return nil, stario.ErrQueueMessageTooLarge
+	}
+	payload := make([]byte, int(length))
+	if _, err := io.ReadFull(conn, payload); err != nil {
+		return nil, err
+	}
+	return payload, nil
+}
+
 func writeBulkDedicatedRecord(conn net.Conn, payload []byte) error {
 	return writeBulkDedicatedRecordWithDeadline(conn, payload, time.Time{})
 }
@@ -133,9 +173,7 @@ func writeBulkDedicatedRecordWithDeadline(conn net.Conn, payload []byte, deadlin
 		var header [bulkDedicatedRecordHeaderLen]byte
 		copy(header[:4], bulkDedicatedRecordMagic)
 		binary.BigEndian.PutUint32(header[4:8], uint32(len(payload)))
-		buffers := net.Buffers{header[:], payload}
-		_, err := buffers.WriteTo(conn)
-		return err
+		return writeNetBuffersFullUnlocked(conn, net.Buffers{header[:], payload})
 	})
 }

@@ -148,7 +186,7 @@ func readBulkDedicatedRecord(conn net.Conn) ([]byte, error) {
 		return nil, err
 	}
 	if string(header[:4]) != bulkDedicatedRecordMagic {
-		return nil, errBulkFastPayloadInvalid
+		return nil, fmt.Errorf("%w: record magic=%x", errBulkFastPayloadInvalid, header[:4])
 	}
 	size := int(binary.BigEndian.Uint32(header[4:8]))
 	if size < 0 {
@@ -224,51 +262,34 @@ func (c *ClientCommon) sendDedicatedBulkAttachRequest(ctx context.Context, conn
 	if err != nil {
 		return bulkAttachResponse{}, err
 	}
-	queue := stario.NewQueue()
 	msg := TransferMsg{
 		ID:    atomic.AddUint64(&c.msgID, 1),
 		Key:   systemBulkAttachKey,
 		Value: reqPayload,
 		Type:  MSG_SYS_WAIT,
 	}
-	frame, err := encodeDirectSignalFrame(queue, c.sequenceEn, c.msgEn, c.SecretKey, msg)
+	frame, err := encodeDirectSignalFrame(stario.NewQueue(), c.sequenceEn, c.msgEn, c.SecretKey, msg)
 	if err != nil {
 		return bulkAttachResponse{}, err
 	}
 	if err := writeFullToConn(conn, frame); err != nil {
 		return bulkAttachResponse{}, err
 	}
-	replyCh := make(chan Message, 1)
-	readBuf := streamReadBuffer()
-	for {
-		if deadline, ok := ctx.Deadline(); ok {
-			_ = conn.SetReadDeadline(deadline)
-		}
-		n, err := conn.Read(readBuf)
-		if err != nil {
-			return bulkAttachResponse{}, err
-		}
-		parseErr := queue.ParseMessageOwned(readBuf[:n], "bulk-attach", func(msgq stario.MsgQueue) error {
-			transfer, err := decodeDirectSignalPayload(c.sequenceDe, c.msgDe, c.SecretKey, msgq.Msg)
-			if err != nil {
-				return err
-			}
-			replyCh <- Message{
-				ServerConn:  c,
-				TransferMsg: transfer,
-				NetType:     NET_CLIENT,
-			}
-			return nil
-		})
-		if parseErr != nil {
-			return bulkAttachResponse{}, parseErr
-		}
-		select {
-		case reply := <-replyCh:
-			return decodeBulkAttachResponse(c.sequenceDe, reply.Value)
-		default:
-		}
+	if deadline, ok := ctx.Deadline(); ok {
+		_ = conn.SetReadDeadline(deadline)
 	}
+	replyPayload, err := readDirectSignalFramePayload(conn)
+	if err != nil {
+		return bulkAttachResponse{}, err
+	}
+	transfer, err := decodeDirectSignalPayload(c.sequenceDe, c.msgDe, c.SecretKey, replyPayload)
+	if err != nil {
+		return bulkAttachResponse{}, err
+	}
+	if transfer.Key != systemBulkAttachKey || transfer.Type != MSG_SYS_REPLY || transfer.ID != msg.ID {
+		return bulkAttachResponse{}, errors.New("invalid bulk attach reply")
+	}
+	return decodeBulkAttachResponse(c.sequenceDe, transfer.Value)
 }

 func (c *ClientCommon) readDedicatedBulkLoop(bulk *bulkHandle, conn net.Conn) {
@@ -323,14 +344,13 @@ func (s *ServerCommon) handleBulkAttachSystemMessage(message Message) bool {
 	}
 	if err != nil {
 		resp.Error = err.Error()
-	} else {
-		resp.Accepted = true
+		if current != nil {
+			_ = s.replyDedicatedBulkAttach(current, message, resp)
+		}
+		return true
 	}
 	if current != nil {
-		_ = s.replyDedicatedBulkAttach(current, message, resp)
-	}
-	if err == nil {
-		if attachErr := s.finishInboundDedicatedBulkAttach(current, logical, bulk); attachErr != nil {
+		if attachErr := s.finishInboundDedicatedBulkAttach(current, logical, bulk, message); attachErr != nil {
 			bulk.markReset(attachErr)
 		}
 	}
@@ -368,7 +388,7 @@ func (s *ServerCommon) resolveInboundDedicatedBulk(current *LogicalConn, req bul
 	return logical, bulk, nil
 }

-func (s *ServerCommon) finishInboundDedicatedBulkAttach(current *LogicalConn, logical *LogicalConn, bulk *bulkHandle) error {
+func (s *ServerCommon) finishInboundDedicatedBulkAttach(current *LogicalConn, logical *LogicalConn, bulk *bulkHandle, message Message) error {
 	if current == nil || logical == nil || bulk == nil {
 		return errBulkLogicalConnNil
 	}
@@ -376,18 +396,56 @@ func (s *ServerCommon) finishInboundDedicatedBulkAttach(current *LogicalConn, lo
 	if err != nil {
 		return err
 	}
-	if err := bulk.attachDedicatedConn(conn); err != nil {
+	fail := func(reason string, err error) error {
 		if conn != nil {
 			_ = conn.Close()
 		}
+		current.markSessionStopped(reason, err)
+		s.removeLogical(current)
 		return err
 	}
+	if err := s.replyDedicatedBulkAttachDetached(current, conn, message, bulkAttachResponse{Accepted: true}); err != nil {
+		return fail("bulk dedicated attach reply failed", err)
+	}
+	if err := bulk.attachDedicatedConn(conn); err != nil {
+		return fail("bulk dedicated attach failed", err)
+	}
 	go s.readDedicatedBulkLoop(logical, bulk, conn)
 	current.markSessionStopped("bulk dedicated attach", nil)
 	s.removeLogical(current)
 	return nil
 }

+func (s *ServerCommon) replyDedicatedBulkAttachDetached(client *LogicalConn, conn net.Conn, message Message, resp bulkAttachResponse) error {
+	if s == nil || client == nil {
+		return errBulkServerNil
+	}
+	if conn == nil {
+		return net.ErrClosed
+	}
+	msgEn := client.msgEnSnapshot()
+	if msgEn == nil {
+		return errTransportPayloadEncryptFailed
+	}
+	encoded, err := s.sequenceEn(resp)
+	if err != nil {
+		return err
+	}
+	reply := TransferMsg{
+		ID:    message.ID,
+		Key:   systemBulkAttachKey,
+		Value: encoded,
+		Type:  MSG_SYS_REPLY,
+	}
+	frame, err := encodeDirectSignalFrame(stario.NewQueue(), s.sequenceEn, msgEn, client.secretKeySnapshot(), reply)
+	if err != nil {
+		return err
+	}
+	return withRawConnWriteLockDeadline(conn, writeDeadlineFromTimeout(client.maxWriteTimeoutSnapshot()), func(conn net.Conn) error {
+		return writeFullToConnUnlocked(conn, frame)
+	})
+}
+
 func (s *ServerCommon) replyDedicatedBulkAttach(client *LogicalConn, message Message, resp bulkAttachResponse) error {
 	if s == nil || client == nil {
 		return errBulkServerNil