fix: close stream adaptive gaps and switch notify to stario v0.1.1

- make stream fast path honor adaptive soft payload limits end-to-end - split oversized fast-stream payloads into sequential frames before batching - use adaptive soft cap when encoding stream batch payloads - move timeout-like error detection into production code for adaptive tx - tune notify FrameReader read size explicitly to avoid throughput regression - drop local stario replace and depend on released b612.me/stario v0.1.1
2026-04-18 16:05:57 +08:00
parent 4f760f2807
commit f038a89771
76 changed files with 12656 additions and 906 deletions
@@ -120,32 +120,145 @@ func decodeBulkFastDataFrame(payload []byte) (bulkFastDataFrame, bool, error) {
 }

 func (c *ClientCommon) encodeFastBulkDataPayload(dataID uint64, seq uint64, chunk []byte) ([]byte, error) {
-	if c != nil && c.fastBulkEncode != nil {
-		return c.fastBulkEncode(c.SecretKey, dataID, seq, chunk)
-	}
-	scratch := getBulkFastFrameScratch(len(chunk))
-	defer putBulkFastFrameScratch(scratch)
-	frame := scratch[:bulkFastPayloadHeaderLen+len(chunk)]
-	if err := encodeBulkFastDataFrameHeader(frame, dataID, seq, len(chunk)); err != nil {
-		return nil, err
-	}
-	copy(frame[bulkFastPayloadHeaderLen:], chunk)
-	return c.encryptTransportPayload(frame)
+	return c.encodeBulkFastPayload(bulkFastFrame{
+		Type:    bulkFastPayloadTypeData,
+		DataID:  dataID,
+		Seq:     seq,
+		Payload: chunk,
+	})
 }

-func (c *ClientCommon) sendFastBulkData(ctx context.Context, dataID uint64, seq uint64, chunk []byte) error {
+func (c *ClientCommon) encodeBulkFastPayload(frame bulkFastFrame) ([]byte, error) {
+	if c == nil {
+		return nil, errBulkClientNil
+	}
+	if c.fastPlainEncode != nil {
+		return encodeBulkFastFramePayloadFast(c.fastPlainEncode, c.SecretKey, frame)
+	}
+	plain, err := encodeBulkFastFramePayload(frame)
+	if err != nil {
+		return nil, err
+	}
+	return c.encryptTransportPayload(plain)
+}
+
+func (c *ClientCommon) encodeBulkFastBatchPayload(frames []bulkFastFrame) ([]byte, error) {
+	if c == nil {
+		return nil, errBulkClientNil
+	}
+	if c.fastPlainEncode != nil {
+		return encodeBulkFastBatchPayloadFast(c.fastPlainEncode, c.SecretKey, frames)
+	}
+	plain, err := encodeBulkFastBatchPlain(frames)
+	if err != nil {
+		return nil, err
+	}
+	return c.encryptTransportPayload(plain)
+}
+
+func (c *ClientCommon) encodeBulkFastPayloadPooled(frame bulkFastFrame) ([]byte, func(), error) {
+	if c == nil {
+		return nil, nil, errBulkClientNil
+	}
+	if runtime := c.modernPSKRuntime; runtime != nil {
+		return encodeBulkFastFramePayloadPooled(runtime, frame)
+	}
+	if c.fastPlainEncode != nil {
+		payload, err := encodeBulkFastFramePayloadFast(c.fastPlainEncode, c.SecretKey, frame)
+		return payload, nil, err
+	}
+	plain, err := encodeBulkFastFramePayload(frame)
+	if err != nil {
+		return nil, nil, err
+	}
+	payload, err := c.encryptTransportPayload(plain)
+	return payload, nil, err
+}
+
+func (c *ClientCommon) encodeBulkFastBatchPayloadPooled(frames []bulkFastFrame) ([]byte, func(), error) {
+	if c == nil {
+		return nil, nil, errBulkClientNil
+	}
+	if runtime := c.modernPSKRuntime; runtime != nil {
+		return encodeBulkFastBatchPayloadPooled(runtime, frames)
+	}
+	if c.fastPlainEncode != nil {
+		payload, err := encodeBulkFastBatchPayloadFast(c.fastPlainEncode, c.SecretKey, frames)
+		return payload, nil, err
+	}
+	plain, err := encodeBulkFastBatchPlain(frames)
+	if err != nil {
+		return nil, nil, err
+	}
+	payload, err := c.encryptTransportPayload(plain)
+	return payload, nil, err
+}
+
+func (c *ClientCommon) sendFastBulkData(ctx context.Context, dataID uint64, seq uint64, chunk []byte, fastPathVersion uint8) error {
+	binding := c.clientTransportBindingSnapshot()
+	if binding == nil {
+		return net.ErrClosed
+	}
+	if sender := binding.clientBulkBatchSenderSnapshot(c); sender != nil {
+		return sender.submitData(ctx, dataID, seq, fastPathVersion, chunk)
+	}
 	payload, err := c.encodeFastBulkDataPayload(dataID, seq, chunk)
 	if err != nil {
 		return err
 	}
+	return c.writePayloadToTransport(payload)
+}
+
+func (c *ClientCommon) sendFastBulkWrite(ctx context.Context, dataID uint64, startSeq uint64, chunkSize int, fastPathVersion uint8, payload []byte, payloadOwned bool) (int, error) {
+	if len(payload) == 0 {
+		return 0, nil
+	}
+	binding := c.clientTransportBindingSnapshot()
+	if binding == nil {
+		return 0, net.ErrClosed
+	}
+	if sender := binding.clientBulkBatchSenderSnapshot(c); sender != nil {
+		return sender.submitWrite(ctx, dataID, startSeq, fastPathVersion, payload, chunkSize, payloadOwned)
+	}
+	if chunkSize <= 0 {
+		chunkSize = defaultBulkChunkSize
+	}
+	written := 0
+	seq := startSeq
+	for written < len(payload) {
+		end := written + chunkSize
+		if end > len(payload) {
+			end = len(payload)
+		}
+		if err := c.sendFastBulkData(ctx, dataID, seq, payload[written:end], fastPathVersion); err != nil {
+			return written, err
+		}
+		seq++
+		written = end
+	}
+	return written, nil
+}
+
+func (c *ClientCommon) sendFastBulkControl(ctx context.Context, frameType uint8, flags uint8, dataID uint64, seq uint64, fastPathVersion uint8, payload []byte) error {
+	frame := bulkFastFrame{
+		Type:    frameType,
+		Flags:   flags,
+		DataID:  dataID,
+		Seq:     seq,
+		Payload: payload,
+	}
 	binding := c.clientTransportBindingSnapshot()
 	if binding == nil {
 		return net.ErrClosed
 	}
-	if sender := binding.bulkBatchSenderSnapshot(); sender != nil {
-		return sender.submit(ctx, payload)
+	if sender := binding.clientBulkBatchSenderSnapshot(c); sender != nil {
+		return sender.submitControl(ctx, frameType, flags, dataID, seq, fastPathVersion, payload)
 	}
-	return c.writePayloadToTransport(payload)
+	encoded, err := c.encodeBulkFastPayload(frame)
+	if err != nil {
+		return err
+	}
+	return c.writePayloadToTransport(encoded)
 }

 func (c *ClientCommon) encodeBulkFastControlPayload(frameType uint8, flags uint8, dataID uint64, seq uint64, payload []byte) ([]byte, error) {
@@ -157,22 +270,81 @@ func (c *ClientCommon) encodeBulkFastControlPayload(frameType uint8, flags uint8
 }

 func (s *ServerCommon) encodeFastBulkDataPayloadLogical(logical *LogicalConn, dataID uint64, seq uint64, chunk []byte) ([]byte, error) {
-	if logical != nil {
-		if fastBulkEncode := logical.fastBulkEncodeSnapshot(); fastBulkEncode != nil {
-			return fastBulkEncode(logical.secretKeySnapshot(), dataID, seq, chunk)
-		}
-	}
-	scratch := getBulkFastFrameScratch(len(chunk))
-	defer putBulkFastFrameScratch(scratch)
-	frame := scratch[:bulkFastPayloadHeaderLen+len(chunk)]
-	if err := encodeBulkFastDataFrameHeader(frame, dataID, seq, len(chunk)); err != nil {
-		return nil, err
-	}
-	copy(frame[bulkFastPayloadHeaderLen:], chunk)
-	return s.encryptTransportPayloadLogical(logical, frame)
+	return s.encodeBulkFastPayloadLogical(logical, bulkFastFrame{
+		Type:    bulkFastPayloadTypeData,
+		DataID:  dataID,
+		Seq:     seq,
+		Payload: chunk,
+	})
 }

-func (s *ServerCommon) sendFastBulkDataTransport(ctx context.Context, logical *LogicalConn, transport *TransportConn, dataID uint64, seq uint64, chunk []byte) error {
+func (s *ServerCommon) encodeBulkFastPayloadLogical(logical *LogicalConn, frame bulkFastFrame) ([]byte, error) {
+	if logical == nil {
+		return nil, errTransportDetached
+	}
+	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
+		return encodeBulkFastFramePayloadFast(fastPlainEncode, logical.secretKeySnapshot(), frame)
+	}
+	plain, err := encodeBulkFastFramePayload(frame)
+	if err != nil {
+		return nil, err
+	}
+	return s.encryptTransportPayloadLogical(logical, plain)
+}
+
+func (s *ServerCommon) encodeBulkFastBatchPayloadLogical(logical *LogicalConn, frames []bulkFastFrame) ([]byte, error) {
+	if logical == nil {
+		return nil, errTransportDetached
+	}
+	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
+		return encodeBulkFastBatchPayloadFast(fastPlainEncode, logical.secretKeySnapshot(), frames)
+	}
+	plain, err := encodeBulkFastBatchPlain(frames)
+	if err != nil {
+		return nil, err
+	}
+	return s.encryptTransportPayloadLogical(logical, plain)
+}
+
+func (s *ServerCommon) encodeBulkFastPayloadLogicalPooled(logical *LogicalConn, frame bulkFastFrame) ([]byte, func(), error) {
+	if logical == nil {
+		return nil, nil, errTransportDetached
+	}
+	if runtime := logical.modernPSKRuntimeSnapshot(); runtime != nil {
+		return encodeBulkFastFramePayloadPooled(runtime, frame)
+	}
+	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
+		payload, err := encodeBulkFastFramePayloadFast(fastPlainEncode, logical.secretKeySnapshot(), frame)
+		return payload, nil, err
+	}
+	plain, err := encodeBulkFastFramePayload(frame)
+	if err != nil {
+		return nil, nil, err
+	}
+	payload, err := s.encryptTransportPayloadLogical(logical, plain)
+	return payload, nil, err
+}
+
+func (s *ServerCommon) encodeBulkFastBatchPayloadLogicalPooled(logical *LogicalConn, frames []bulkFastFrame) ([]byte, func(), error) {
+	if logical == nil {
+		return nil, nil, errTransportDetached
+	}
+	if runtime := logical.modernPSKRuntimeSnapshot(); runtime != nil {
+		return encodeBulkFastBatchPayloadPooled(runtime, frames)
+	}
+	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
+		payload, err := encodeBulkFastBatchPayloadFast(fastPlainEncode, logical.secretKeySnapshot(), frames)
+		return payload, nil, err
+	}
+	plain, err := encodeBulkFastBatchPlain(frames)
+	if err != nil {
+		return nil, nil, err
+	}
+	payload, err := s.encryptTransportPayloadLogical(logical, plain)
+	return payload, nil, err
+}
+
+func (s *ServerCommon) sendFastBulkDataTransport(ctx context.Context, logical *LogicalConn, transport *TransportConn, dataID uint64, seq uint64, chunk []byte, fastPathVersion uint8) error {
 	if err := s.ensureServerTransportSendReady(transport); err != nil {
 		return err
 	}
@@ -182,18 +354,87 @@ func (s *ServerCommon) sendFastBulkDataTransport(ctx context.Context, logical *L
 	if logical == nil {
 		return errTransportDetached
 	}
+	if binding := logical.transportBindingSnapshot(); binding != nil {
+		if binding.queueSnapshot() != nil {
+			if sender := binding.serverBulkBatchSenderSnapshot(logical); sender != nil {
+				return sender.submitData(ctx, dataID, seq, fastPathVersion, chunk)
+			}
+		}
+	}
 	payload, err := s.encodeFastBulkDataPayloadLogical(logical, dataID, seq, chunk)
 	if err != nil {
 		return err
 	}
+	return s.writeEnvelopePayload(logical, transport, nil, payload)
+}
+
+func (s *ServerCommon) sendFastBulkWriteTransport(ctx context.Context, logical *LogicalConn, transport *TransportConn, dataID uint64, startSeq uint64, chunkSize int, fastPathVersion uint8, payload []byte, payloadOwned bool) (int, error) {
+	if len(payload) == 0 {
+		return 0, nil
+	}
+	if err := s.ensureServerTransportSendReady(transport); err != nil {
+		return 0, err
+	}
+	if logical == nil && transport != nil {
+		logical = transport.logicalConnSnapshot()
+	}
+	if logical == nil {
+		return 0, errTransportDetached
+	}
 	if binding := logical.transportBindingSnapshot(); binding != nil {
 		if binding.queueSnapshot() != nil {
-			if sender := binding.bulkBatchSenderSnapshot(); sender != nil {
-				return sender.submit(ctx, payload)
+			if sender := binding.serverBulkBatchSenderSnapshot(logical); sender != nil {
+				return sender.submitWrite(ctx, dataID, startSeq, fastPathVersion, payload, chunkSize, payloadOwned)
 			}
 		}
 	}
-	return s.writeEnvelopePayload(logical, transport, nil, payload)
+	if chunkSize <= 0 {
+		chunkSize = defaultBulkChunkSize
+	}
+	written := 0
+	seq := startSeq
+	for written < len(payload) {
+		end := written + chunkSize
+		if end > len(payload) {
+			end = len(payload)
+		}
+		if err := s.sendFastBulkDataTransport(ctx, logical, transport, dataID, seq, payload[written:end], fastPathVersion); err != nil {
+			return written, err
+		}
+		seq++
+		written = end
+	}
+	return written, nil
+}
+
+func (s *ServerCommon) sendFastBulkControlTransport(ctx context.Context, logical *LogicalConn, transport *TransportConn, frameType uint8, flags uint8, dataID uint64, seq uint64, fastPathVersion uint8, payload []byte) error {
+	if err := s.ensureServerTransportSendReady(transport); err != nil {
+		return err
+	}
+	if logical == nil && transport != nil {
+		logical = transport.logicalConnSnapshot()
+	}
+	if logical == nil {
+		return errTransportDetached
+	}
+	if binding := logical.transportBindingSnapshot(); binding != nil {
+		if binding.queueSnapshot() != nil {
+			if sender := binding.serverBulkBatchSenderSnapshot(logical); sender != nil {
+				return sender.submitControl(ctx, frameType, flags, dataID, seq, fastPathVersion, payload)
+			}
+		}
+	}
+	encoded, err := s.encodeBulkFastPayloadLogical(logical, bulkFastFrame{
+		Type:    frameType,
+		Flags:   flags,
+		DataID:  dataID,
+		Seq:     seq,
+		Payload: payload,
+	})
+	if err != nil {
+		return err
+	}
+	return s.writeEnvelopePayload(logical, transport, nil, encoded)
 }

 func (s *ServerCommon) encodeBulkFastControlPayloadLogical(logical *LogicalConn, frameType uint8, flags uint8, dataID uint64, seq uint64, payload []byte) ([]byte, error) {
@@ -224,18 +465,22 @@ func (c *ClientCommon) dispatchInboundTransportPayload(payload []byte, now time.
 	if err != nil {
 		return err
 	}
-	if frame, matched, err := decodeBulkFastFrame(plain); matched {
+	if frames, matched, err := decodeBulkFastFrames(plain); matched {
 		if err != nil {
 			return err
 		}
-		c.dispatchFastBulkFrame(frame)
+		for _, frame := range frames {
+			c.dispatchFastBulkFrame(frame)
+		}
 		return nil
 	}
-	if frame, matched, err := decodeStreamFastDataFrame(plain); matched {
+	if frames, matched, err := decodeStreamFastDataFrames(plain); matched {
 		if err != nil {
 			return err
 		}
-		c.dispatchFastStreamData(frame)
+		for _, frame := range frames {
+			c.dispatchFastStreamData(frame)
+		}
 		return nil
 	}
 	env, err := c.decodeEnvelopePlain(plain)
@@ -257,18 +502,22 @@ func (s *ServerCommon) dispatchInboundTransportPayload(logical *LogicalConn, tra
 	if err != nil {
 		return err
 	}
-	if frame, matched, err := decodeBulkFastFrame(plain); matched {
+	if frames, matched, err := decodeBulkFastFrames(plain); matched {
 		if err != nil {
 			return err
 		}
-		s.dispatchFastBulkFrame(logical, transport, conn, frame)
+		for _, frame := range frames {
+			s.dispatchFastBulkFrame(logical, transport, conn, frame)
+		}
 		return nil
 	}
-	if frame, matched, err := decodeStreamFastDataFrame(plain); matched {
+	if frames, matched, err := decodeStreamFastDataFrames(plain); matched {
 		if err != nil {
 			return err
 		}
-		s.dispatchFastStreamData(logical, transport, conn, frame)
+		for _, frame := range frames {
+			s.dispatchFastStreamData(logical, transport, conn, frame)
+		}
 		return nil
 	}
 	env, err := s.decodeEnvelopePlain(plain)