fix: close stream adaptive gaps and switch notify to stario v0.1.1

- make stream fast path honor adaptive soft payload limits end-to-end
  - split oversized fast-stream payloads into sequential frames before batching
  - use adaptive soft cap when encoding stream batch payloads
  - move timeout-like error detection into production code for adaptive tx
  - tune notify FrameReader read size explicitly to avoid throughput regression
  - drop local stario replace and depend on released b612.me/stario v0.1.1
This commit is contained in:
2026-04-18 16:05:57 +08:00
parent 4f760f2807
commit f038a89771
76 changed files with 12656 additions and 906 deletions
+397 -29
View File
@@ -12,14 +12,18 @@ import (
)
const (
bulkDedicatedBatchMagic = "NBD2"
bulkDedicatedBatchVersion = 1
bulkDedicatedBatchHeaderLen = 20
bulkDedicatedBatchItemHeaderLen = 16
bulkDedicatedBatchMaxItems = 32
bulkDedicatedBatchMaxPlainBytes = 8 * 1024 * 1024
bulkDedicatedSendQueueSize = bulkDedicatedBatchMaxItems
bulkDedicatedReleasePayloadLen = 12
bulkDedicatedBatchMagic = "NBD2"
bulkDedicatedBatchVersion = 1
bulkDedicatedBatchHeaderLen = 20
bulkDedicatedBatchItemHeaderLen = 16
bulkDedicatedSuperBatchMagic = "NBD3"
bulkDedicatedSuperBatchVersion = 1
bulkDedicatedSuperBatchHeaderLen = 12
bulkDedicatedSuperBatchGroupHeaderLen = 12
bulkDedicatedBatchMaxItems = 64
bulkDedicatedBatchMaxPlainBytes = 16 * 1024 * 1024
bulkDedicatedSendQueueSize = bulkDedicatedBatchMaxItems
bulkDedicatedReleasePayloadLen = 12
)
const (
@@ -46,6 +50,16 @@ type bulkDedicatedSendRequest struct {
Payload []byte
}
type bulkDedicatedOutboundBatch struct {
DataID uint64
Items []bulkDedicatedSendRequest
}
type bulkDedicatedInboundBatch struct {
DataID uint64
Items []bulkDedicatedBatchItem
}
type bulkDedicatedBatchRequest struct {
Ctx context.Context
Items []bulkDedicatedSendRequest
@@ -165,7 +179,7 @@ func (s *bulkDedicatedSender) submitWriteBatch(ctx context.Context, items []bulk
if submitted, err := s.tryDirectSubmitBatch(ctx, items); submitted {
return err
}
queuedItems := cloneBulkDedicatedSendRequests(items)
queuedItems := copyBulkDedicatedSendRequests(items)
return s.submitBatch(ctx, queuedItems, true)
}
@@ -471,6 +485,23 @@ func cloneBulkDedicatedSendRequests(items []bulkDedicatedSendRequest) []bulkDedi
return cloned
}
func copyBulkDedicatedSendRequests(items []bulkDedicatedSendRequest) []bulkDedicatedSendRequest {
if len(items) == 0 {
return nil
}
copied := make([]bulkDedicatedSendRequest, len(items))
copy(copied, items)
return copied
}
func bulkDedicatedSendRequestsLen(items []bulkDedicatedSendRequest) int {
total := 0
for _, item := range items {
total += bulkDedicatedSendRequestLen(item)
}
return total
}
func bulkDedicatedSendRequestLen(req bulkDedicatedSendRequest) int {
return bulkDedicatedSendRequestLenFromPayloadLen(len(req.Payload))
}
@@ -479,6 +510,83 @@ func bulkDedicatedSendRequestLenFromPayloadLen(payloadLen int) int {
return bulkDedicatedBatchItemHeaderLen + payloadLen
}
func bulkDedicatedBatchesPlainLen(batches []bulkDedicatedOutboundBatch) int {
switch len(batches) {
case 0:
return 0
case 1:
return bulkDedicatedBatchPlainLen(batches[0].Items)
default:
total := bulkDedicatedSuperBatchHeaderLen
for _, batch := range batches {
total += bulkDedicatedSuperBatchGroupHeaderLen + bulkDedicatedSendRequestsLen(batch.Items)
}
return total
}
}
func encodeBulkDedicatedBatchesPlain(batches []bulkDedicatedOutboundBatch) ([]byte, error) {
if len(batches) == 0 {
return nil, errBulkFastPayloadInvalid
}
total := bulkDedicatedBatchesPlainLen(batches)
buf := make([]byte, total)
if err := writeBulkDedicatedBatchesPlain(buf, batches); err != nil {
return nil, err
}
return buf, nil
}
func writeBulkDedicatedBatchesPlain(buf []byte, batches []bulkDedicatedOutboundBatch) error {
switch len(batches) {
case 0:
return errBulkFastPayloadInvalid
case 1:
return writeBulkDedicatedBatchPlain(buf, batches[0].DataID, batches[0].Items)
default:
return writeBulkDedicatedSuperBatchPlain(buf, batches)
}
}
func writeBulkDedicatedSuperBatchPlain(buf []byte, batches []bulkDedicatedOutboundBatch) error {
if len(batches) <= 1 {
return errBulkFastPayloadInvalid
}
if len(buf) != bulkDedicatedBatchesPlainLen(batches) || len(buf) > bulkDedicatedBatchMaxPlainBytes {
return errBulkFastPayloadInvalid
}
copy(buf[:4], bulkDedicatedSuperBatchMagic)
buf[4] = bulkDedicatedSuperBatchVersion
binary.BigEndian.PutUint32(buf[8:12], uint32(len(batches)))
offset := bulkDedicatedSuperBatchHeaderLen
totalItems := 0
for _, batch := range batches {
if batch.DataID == 0 || len(batch.Items) == 0 {
return errBulkFastPayloadInvalid
}
totalItems += len(batch.Items)
if totalItems > bulkDedicatedBatchMaxItems {
return errBulkFastPayloadInvalid
}
binary.BigEndian.PutUint64(buf[offset:offset+8], batch.DataID)
binary.BigEndian.PutUint32(buf[offset+8:offset+12], uint32(len(batch.Items)))
offset += bulkDedicatedSuperBatchGroupHeaderLen
for _, item := range batch.Items {
buf[offset] = item.Type
buf[offset+1] = item.Flags
binary.BigEndian.PutUint64(buf[offset+4:offset+12], item.Seq)
binary.BigEndian.PutUint32(buf[offset+12:offset+16], uint32(len(item.Payload)))
offset += bulkDedicatedBatchItemHeaderLen
copy(buf[offset:offset+len(item.Payload)], item.Payload)
offset += len(item.Payload)
}
}
if offset != len(buf) {
return errBulkFastPayloadInvalid
}
return nil
}
func encodeBulkDedicatedReleasePayload(bytes int64, chunks int) ([]byte, error) {
if bytes <= 0 && chunks <= 0 {
return nil, errBulkFastPayloadInvalid
@@ -505,24 +613,26 @@ func decodeBulkDedicatedReleasePayload(payload []byte) (int64, int, error) {
}
func encodeBulkDedicatedBatchPlain(dataID uint64, items []bulkDedicatedSendRequest) ([]byte, error) {
if dataID == 0 || len(items) == 0 {
return nil, errBulkFastPayloadInvalid
}
total := bulkDedicatedBatchPlainLen(items)
buf := make([]byte, total)
if err := writeBulkDedicatedBatchPlain(buf, dataID, items); err != nil {
return nil, err
}
return buf, nil
return encodeBulkDedicatedBatchesPlain([]bulkDedicatedOutboundBatch{{
DataID: dataID,
Items: items,
}})
}
func encodeBulkDedicatedBatchPayloadFast(encode transportFastPlainEncoder, secretKey []byte, dataID uint64, items []bulkDedicatedSendRequest) ([]byte, error) {
return encodeBulkDedicatedBatchesPayloadFast(encode, secretKey, []bulkDedicatedOutboundBatch{{
DataID: dataID,
Items: items,
}})
}
func encodeBulkDedicatedBatchesPayloadFast(encode transportFastPlainEncoder, secretKey []byte, batches []bulkDedicatedOutboundBatch) ([]byte, error) {
if encode == nil {
return nil, errTransportPayloadEncryptFailed
}
plainLen := bulkDedicatedBatchPlainLen(items)
plainLen := bulkDedicatedBatchesPlainLen(batches)
return encode(secretKey, plainLen, func(dst []byte) error {
return writeBulkDedicatedBatchPlain(dst, dataID, items)
return writeBulkDedicatedBatchesPlain(dst, batches)
})
}
@@ -606,29 +716,268 @@ func decodeBulkDedicatedBatchPlain(payload []byte) (uint64, []bulkDedicatedBatch
return dataID, items, true, nil
}
func decodeDedicatedBulkInboundItems(expectedDataID uint64, plain []byte) ([]bulkDedicatedBatchItem, error) {
func decodeBulkDedicatedSuperBatchPlain(payload []byte) ([]bulkDedicatedInboundBatch, bool, error) {
if len(payload) < 4 || string(payload[:4]) != bulkDedicatedSuperBatchMagic {
return nil, false, nil
}
if len(payload) < bulkDedicatedSuperBatchHeaderLen {
return nil, true, errBulkFastPayloadInvalid
}
if payload[4] != bulkDedicatedSuperBatchVersion {
return nil, true, errBulkFastPayloadInvalid
}
groupCount := int(binary.BigEndian.Uint32(payload[8:12]))
if groupCount <= 0 {
return nil, true, errBulkFastPayloadInvalid
}
batches := make([]bulkDedicatedInboundBatch, 0, groupCount)
offset := bulkDedicatedSuperBatchHeaderLen
totalItems := 0
for i := 0; i < groupCount; i++ {
if len(payload)-offset < bulkDedicatedSuperBatchGroupHeaderLen {
return nil, true, errBulkFastPayloadInvalid
}
dataID := binary.BigEndian.Uint64(payload[offset : offset+8])
count := int(binary.BigEndian.Uint32(payload[offset+8 : offset+12]))
offset += bulkDedicatedSuperBatchGroupHeaderLen
if dataID == 0 || count <= 0 {
return nil, true, errBulkFastPayloadInvalid
}
totalItems += count
if totalItems > bulkDedicatedBatchMaxItems {
return nil, true, errBulkFastPayloadInvalid
}
items := make([]bulkDedicatedBatchItem, 0, count)
for j := 0; j < count; j++ {
if len(payload)-offset < bulkDedicatedBatchItemHeaderLen {
return nil, true, errBulkFastPayloadInvalid
}
itemType := payload[offset]
switch itemType {
case bulkFastPayloadTypeData, bulkFastPayloadTypeClose, bulkFastPayloadTypeReset, bulkFastPayloadTypeRelease:
default:
return nil, true, errBulkFastPayloadInvalid
}
flags := payload[offset+1]
seq := binary.BigEndian.Uint64(payload[offset+4 : offset+12])
dataLen := int(binary.BigEndian.Uint32(payload[offset+12 : offset+16]))
offset += bulkDedicatedBatchItemHeaderLen
if dataLen < 0 || len(payload)-offset < dataLen {
return nil, true, errBulkFastPayloadInvalid
}
items = append(items, bulkDedicatedBatchItem{
Type: itemType,
Flags: flags,
Seq: seq,
Payload: payload[offset : offset+dataLen],
})
offset += dataLen
}
batches = append(batches, bulkDedicatedInboundBatch{
DataID: dataID,
Items: items,
})
}
if offset != len(payload) {
return nil, true, errBulkFastPayloadInvalid
}
return batches, true, nil
}
func decodeDedicatedBulkInboundBatches(plain []byte) ([]bulkDedicatedInboundBatch, error) {
if batches, matched, err := decodeBulkDedicatedSuperBatchPlain(plain); matched {
if err != nil {
return nil, err
}
return batches, nil
}
if dataID, items, matched, err := decodeBulkDedicatedBatchPlain(plain); matched {
if err != nil {
return nil, err
}
if expectedDataID == 0 || dataID != expectedDataID {
return nil, errBulkFastPayloadInvalid
}
return items, nil
return []bulkDedicatedInboundBatch{{
DataID: dataID,
Items: items,
}}, nil
}
frame, matched, err := decodeBulkFastFrame(plain)
if err != nil {
return nil, err
}
if !matched || expectedDataID == 0 || frame.DataID != expectedDataID {
if !matched || frame.DataID == 0 {
return nil, errBulkFastPayloadInvalid
}
return []bulkDedicatedBatchItem{{
return []bulkDedicatedInboundBatch{{
DataID: frame.DataID,
Items: []bulkDedicatedBatchItem{{
Type: frame.Type,
Flags: frame.Flags,
Seq: frame.Seq,
Payload: frame.Payload,
}},
}}, nil
}
func decodeDedicatedBulkInboundPayload(plain []byte) (uint64, []bulkDedicatedBatchItem, error) {
batches, err := decodeDedicatedBulkInboundBatches(plain)
if err != nil {
return 0, nil, err
}
if len(batches) != 1 {
return 0, nil, errBulkFastPayloadInvalid
}
return batches[0].DataID, batches[0].Items, nil
}
func decodeDedicatedBulkInboundItems(expectedDataID uint64, plain []byte) ([]bulkDedicatedBatchItem, error) {
dataID, items, err := decodeDedicatedBulkInboundPayload(plain)
if err != nil {
return nil, err
}
if expectedDataID == 0 || dataID != expectedDataID {
return nil, errBulkFastPayloadInvalid
}
return items, nil
}
func hasBulkDedicatedMagic(payload []byte, magic string) bool {
return len(payload) >= 4 &&
payload[0] == magic[0] &&
payload[1] == magic[1] &&
payload[2] == magic[2] &&
payload[3] == magic[3]
}
func walkDedicatedBulkInboundPayload(plain []byte, visit func(dataID uint64, item bulkDedicatedBatchItem) error) error {
if visit == nil {
return errBulkFastPayloadInvalid
}
if hasBulkDedicatedMagic(plain, bulkDedicatedSuperBatchMagic) {
return walkDedicatedBulkInboundSuperBatchPlain(plain, visit)
}
if hasBulkDedicatedMagic(plain, bulkDedicatedBatchMagic) {
return walkDedicatedBulkInboundBatchPlain(plain, visit)
}
frame, matched, err := decodeBulkFastFrame(plain)
if err != nil {
return err
}
if !matched || frame.DataID == 0 {
return errBulkFastPayloadInvalid
}
return visit(frame.DataID, bulkDedicatedBatchItem{
Type: frame.Type,
Flags: frame.Flags,
Seq: frame.Seq,
Payload: frame.Payload,
}}, nil
})
}
func walkDedicatedBulkInboundBatchPlain(payload []byte, visit func(dataID uint64, item bulkDedicatedBatchItem) error) error {
if len(payload) < bulkDedicatedBatchHeaderLen {
return errBulkFastPayloadInvalid
}
if payload[4] != bulkDedicatedBatchVersion {
return errBulkFastPayloadInvalid
}
dataID := binary.BigEndian.Uint64(payload[8:16])
count := int(binary.BigEndian.Uint32(payload[16:20]))
if dataID == 0 || count <= 0 {
return errBulkFastPayloadInvalid
}
offset := bulkDedicatedBatchHeaderLen
for i := 0; i < count; i++ {
if len(payload)-offset < bulkDedicatedBatchItemHeaderLen {
return errBulkFastPayloadInvalid
}
itemType := payload[offset]
switch itemType {
case bulkFastPayloadTypeData, bulkFastPayloadTypeClose, bulkFastPayloadTypeReset, bulkFastPayloadTypeRelease:
default:
return errBulkFastPayloadInvalid
}
flags := payload[offset+1]
seq := binary.BigEndian.Uint64(payload[offset+4 : offset+12])
dataLen := int(binary.BigEndian.Uint32(payload[offset+12 : offset+16]))
offset += bulkDedicatedBatchItemHeaderLen
if dataLen < 0 || len(payload)-offset < dataLen {
return errBulkFastPayloadInvalid
}
if err := visit(dataID, bulkDedicatedBatchItem{
Type: itemType,
Flags: flags,
Seq: seq,
Payload: payload[offset : offset+dataLen],
}); err != nil {
return err
}
offset += dataLen
}
if offset != len(payload) {
return errBulkFastPayloadInvalid
}
return nil
}
func walkDedicatedBulkInboundSuperBatchPlain(payload []byte, visit func(dataID uint64, item bulkDedicatedBatchItem) error) error {
if len(payload) < bulkDedicatedSuperBatchHeaderLen {
return errBulkFastPayloadInvalid
}
if payload[4] != bulkDedicatedSuperBatchVersion {
return errBulkFastPayloadInvalid
}
groupCount := int(binary.BigEndian.Uint32(payload[8:12]))
if groupCount <= 0 {
return errBulkFastPayloadInvalid
}
offset := bulkDedicatedSuperBatchHeaderLen
totalItems := 0
for i := 0; i < groupCount; i++ {
if len(payload)-offset < bulkDedicatedSuperBatchGroupHeaderLen {
return errBulkFastPayloadInvalid
}
dataID := binary.BigEndian.Uint64(payload[offset : offset+8])
count := int(binary.BigEndian.Uint32(payload[offset+8 : offset+12]))
offset += bulkDedicatedSuperBatchGroupHeaderLen
if dataID == 0 || count <= 0 {
return errBulkFastPayloadInvalid
}
totalItems += count
if totalItems > bulkDedicatedBatchMaxItems {
return errBulkFastPayloadInvalid
}
for j := 0; j < count; j++ {
if len(payload)-offset < bulkDedicatedBatchItemHeaderLen {
return errBulkFastPayloadInvalid
}
itemType := payload[offset]
switch itemType {
case bulkFastPayloadTypeData, bulkFastPayloadTypeClose, bulkFastPayloadTypeReset, bulkFastPayloadTypeRelease:
default:
return errBulkFastPayloadInvalid
}
flags := payload[offset+1]
seq := binary.BigEndian.Uint64(payload[offset+4 : offset+12])
dataLen := int(binary.BigEndian.Uint32(payload[offset+12 : offset+16]))
offset += bulkDedicatedBatchItemHeaderLen
if dataLen < 0 || len(payload)-offset < dataLen {
return errBulkFastPayloadInvalid
}
if err := visit(dataID, bulkDedicatedBatchItem{
Type: itemType,
Flags: flags,
Seq: seq,
Payload: payload[offset : offset+dataLen],
}); err != nil {
return err
}
offset += dataLen
}
}
if offset != len(payload) {
return errBulkFastPayloadInvalid
}
return nil
}
func normalizeDedicatedBulkSendError(err error) error {
@@ -643,13 +992,23 @@ func normalizeDedicatedBulkSendError(err error) error {
}
func dispatchDedicatedBulkInboundItem(bulk *bulkHandle, item bulkDedicatedBatchItem) error {
return dispatchDedicatedBulkInboundItemWithRelease(bulk, item, nil)
}
func dispatchDedicatedBulkInboundItemWithRelease(bulk *bulkHandle, item bulkDedicatedBatchItem, release func()) error {
if bulk == nil {
if release != nil {
release()
}
return io.ErrClosedPipe
}
switch item.Type {
case bulkFastPayloadTypeData:
return bulk.pushOwnedChunkNoReset(item.Payload)
return bulk.pushOwnedChunkWithReleaseNoReset(item.Payload, release)
case bulkFastPayloadTypeClose:
if release != nil {
release()
}
if item.Flags&bulkFastPayloadFlagFullClose != 0 {
bulk.markPeerClosed()
return nil
@@ -657,6 +1016,9 @@ func dispatchDedicatedBulkInboundItem(bulk *bulkHandle, item bulkDedicatedBatchI
bulk.markRemoteClosed()
return nil
case bulkFastPayloadTypeReset:
if release != nil {
release()
}
resetErr := errBulkReset
if len(item.Payload) > 0 {
resetErr = bulkRemoteResetError(string(item.Payload))
@@ -664,6 +1026,9 @@ func dispatchDedicatedBulkInboundItem(bulk *bulkHandle, item bulkDedicatedBatchI
bulk.markReset(bulkResetError(resetErr))
return nil
case bulkFastPayloadTypeRelease:
if release != nil {
release()
}
bytes, chunks, err := decodeBulkDedicatedReleasePayload(item.Payload)
if err != nil {
return err
@@ -671,6 +1036,9 @@ func dispatchDedicatedBulkInboundItem(bulk *bulkHandle, item bulkDedicatedBatchI
bulk.releaseOutboundWindow(bytes, chunks)
return nil
default:
if release != nil {
release()
}
return errBulkFastPayloadInvalid
}
}