notify/bulk_dedicated.go

package notify

import (
	"b612.me/notify/internal/transport"
	"b612.me/stario"
	"context"
	cryptorand "crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"
)

const (
	systemBulkAttachKey          = "_notify_bulk_attach"
	bulkDedicatedRecordMagic     = "NBR1"
	bulkDedicatedRecordHeaderLen = 8
	bulkDedicatedRecordMaxBytes  = 20 * 1024 * 1024

	defaultBulkDedicatedAttachLimit   = 16
	defaultBulkDedicatedActiveLimit   = 4096
	defaultBulkDedicatedLaneLimit     = 4
	defaultBulkDedicatedAttachRetry   = 2
	defaultBulkDedicatedAttachBackoff = 150 * time.Millisecond
	defaultBulkDedicatedDialTimeout   = 5 * time.Second
	defaultBulkDedicatedHelloTimeout  = 10 * time.Second
)

type bulkAttachRequest struct {
	PeerID      string
	BulkID      string
	AttachToken string
}

type bulkAttachResponse struct {
	Accepted   bool
	Error      string
	Code       string
	Retryable  bool
	FailedSeq  uint64
	FailedBulk string
}

type bulkAttachErrorCode string

const (
	bulkAttachErrorCodeInvalidRequest    bulkAttachErrorCode = "invalid_request"
	bulkAttachErrorCodeServerUnavailable bulkAttachErrorCode = "server_unavailable"
	bulkAttachErrorCodePeerNotFound      bulkAttachErrorCode = "peer_not_found"
	bulkAttachErrorCodeBulkNotFound      bulkAttachErrorCode = "bulk_not_found"
	bulkAttachErrorCodeBulkNotDedicated  bulkAttachErrorCode = "bulk_not_dedicated"
	bulkAttachErrorCodeTokenMismatch     bulkAttachErrorCode = "token_mismatch"
	bulkAttachErrorCodeAlreadyAttached   bulkAttachErrorCode = "already_attached"
	bulkAttachErrorCodeAttachFailed      bulkAttachErrorCode = "attach_failed"
	bulkAttachErrorCodeInternal          bulkAttachErrorCode = "internal_error"
)

type bulkAttachError struct {
	Code       bulkAttachErrorCode
	Retryable  bool
	Message    string
	FailedSeq  uint64
	FailedBulk string
}

func (e *bulkAttachError) Error() string {
	if e == nil {
		return ""
	}
	if e.Message != "" {
		return e.Message
	}
	if e.Code != "" {
		return string(e.Code)
	}
	return "bulk attach failed"
}

func newBulkAttachError(code bulkAttachErrorCode, retryable bool, message string) *bulkAttachError {
	return &bulkAttachError{
		Code:      code,
		Retryable: retryable,
		Message:   message,
	}
}

func toBulkAttachResponseError(err error, bulkID string) bulkAttachResponse {
	resp := bulkAttachResponse{
		Accepted:   false,
		FailedBulk: bulkID,
	}
	if err == nil {
		resp.Error = "bulk attach failed"
		resp.Code = string(bulkAttachErrorCodeInternal)
		return resp
	}
	var attachErr *bulkAttachError
	if errors.As(err, &attachErr) && attachErr != nil {
		resp.Error = attachErr.Error()
		resp.Code = string(attachErr.Code)
		resp.Retryable = attachErr.Retryable
		resp.FailedSeq = attachErr.FailedSeq
		if attachErr.FailedBulk != "" {
			resp.FailedBulk = attachErr.FailedBulk
		}
		if resp.Code == "" {
			resp.Code = string(bulkAttachErrorCodeInternal)
		}
		return resp
	}
	resp.Error = err.Error()
	resp.Code = string(bulkAttachErrorCodeInternal)
	return resp
}

func newBulkAttachToken() string {
	var buf [16]byte
	if _, err := cryptorand.Read(buf[:]); err == nil {
		return hex.EncodeToString(buf[:])
	}
	return ""
}

func decodeBulkAttachRequest(decodeFn func([]byte) (interface{}, error), data MsgVal) (bulkAttachRequest, error) {
	var req bulkAttachRequest
	if decodeFn == nil {
		decodeFn = Decode
	}
	raw := []byte(data)
	value, err := decodeFn(raw)
	if err != nil {
		return req, err
	}
	switch typed := value.(type) {
	case bulkAttachRequest:
		return typed, nil
	case *bulkAttachRequest:
		if typed == nil {
			return req, errors.New("bulk attach request is nil")
		}
		return *typed, nil
	default:
		return req, errors.New("invalid bulk attach payload")
	}
}

func decodeBulkAttachResponse(decodeFn func([]byte) (interface{}, error), data MsgVal) (bulkAttachResponse, error) {
	var resp bulkAttachResponse
	if decodeFn == nil {
		decodeFn = Decode
	}
	raw := []byte(data)
	value, err := decodeFn(raw)
	if err != nil {
		return resp, err
	}
	switch typed := value.(type) {
	case bulkAttachResponse:
		return typed, nil
	case *bulkAttachResponse:
		if typed == nil {
			return resp, errors.New("bulk attach response is nil")
		}
		return *typed, nil
	default:
		return resp, errors.New("invalid bulk attach response")
	}
}

func encodeDirectSignalFrame(queue *stario.StarQueue, sequenceEn func(interface{}) ([]byte, error), msgEn func([]byte, []byte) []byte, secretKey []byte, msg TransferMsg) ([]byte, error) {
	if queue == nil {
		queue = stario.NewQueue()
	}
	env, err := wrapTransferMsgEnvelope(msg, sequenceEn)
	if err != nil {
		return nil, err
	}
	plain, err := sequenceEn(env)
	if err != nil {
		return nil, err
	}
	payload := msgEn(secretKey, plain)
	if payload == nil && len(plain) != 0 {
		return nil, errTransportPayloadEncryptFailed
	}
	if err := validateTransportFramePayloadLen(payload); err != nil {
		return nil, err
	}
	return queue.BuildMessage(payload), nil
}

func decodeDirectSignalPayload(sequenceDe func([]byte) (interface{}, error), msgDe func([]byte, []byte) []byte, secretKey []byte, payload []byte) (TransferMsg, error) {
	plain := msgDe(secretKey, payload)
	if plain == nil && len(payload) != 0 {
		return TransferMsg{}, errTransportPayloadDecryptFailed
	}
	value, err := sequenceDe(plain)
	if err != nil {
		return TransferMsg{}, err
	}
	env, ok := value.(Envelope)
	if !ok {
		return TransferMsg{}, errors.New("invalid signal envelope")
	}
	return unwrapTransferMsgEnvelope(env, sequenceDe)
}

func readDirectSignalFramePayload(conn net.Conn) ([]byte, error) {
	if conn == nil {
		return nil, net.ErrClosed
	}
	return newTransportFrameReader(conn, stario.NewQueueCtx(nil, 1, transportFrameMaxPayloadBytes)).Next()
}

func writeBulkDedicatedRecord(conn net.Conn, payload []byte) error {
	return writeBulkDedicatedRecordWithDeadline(conn, payload, time.Time{})
}

func writeBulkDedicatedRecordWithDeadline(conn net.Conn, payload []byte, deadline time.Time) error {
	return writeBulkDedicatedRecordWithDeadlineTrace(conn, payload, deadline, 0)
}

func writeBulkDedicatedRecordWithDeadlineTrace(conn net.Conn, payload []byte, deadline time.Time, traceDataID uint64) error {
	if conn == nil {
		return net.ErrClosed
	}
	if len(payload) > bulkDedicatedRecordMaxBytes {
		return fmt.Errorf("%w: dedicated record payload=%d max=%d", errBulkFastPayloadInvalid, len(payload), bulkDedicatedRecordMaxBytes)
	}
	if deadline.IsZero() {
		deadline = writeDeadlineFromTimeout(defaultBulkDataWriteTimeout)
	}
	var prepareStarted time.Time
	if traceDataID != 0 {
		prepareStarted = time.Now()
		fmt.Printf("[bulk-debug] at=%s record write prepare data=%d conn=%T local=%v remote=%v bytes=%d\n", prepareStarted.Format(time.RFC3339Nano), traceDataID, conn, conn.LocalAddr(), conn.RemoteAddr(), len(payload))
		prepareStarted = time.Now()
	}
	return withRawConnWriteLockDeadline(conn, deadline, func(conn net.Conn) error {
		var header [bulkDedicatedRecordHeaderLen]byte
		copy(header[:4], bulkDedicatedRecordMagic)
		binary.BigEndian.PutUint32(header[4:8], uint32(len(payload)))
		if traceDataID == 0 {
			return writeNetBuffersFullUnlocked(conn, net.Buffers{header[:], payload})
		}
		fmt.Printf("[bulk-debug] at=%s record socket write begin data=%d gate-and-deadline=%s\n", time.Now().Format(time.RFC3339Nano), traceDataID, time.Since(prepareStarted))
		writeStarted := time.Now()
		err := writeNetBuffersFullUnlocked(conn, net.Buffers{header[:], payload})
		fmt.Printf("[bulk-debug] at=%s record socket write end data=%d elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), traceDataID, time.Since(writeStarted), err)
		return err
	})
}

func readBulkDedicatedRecord(conn net.Conn) ([]byte, error) {
	payload, release, err := readBulkDedicatedRecordPooled(conn)
	if err != nil {
		return nil, err
	}
	if release != nil {
		defer release()
	}
	return append([]byte(nil), payload...), nil
}

func readBulkDedicatedRecordPooled(conn net.Conn) ([]byte, func(), error) {
	return readBulkDedicatedRecordPooledTrace(conn, false)
}

func readBulkDedicatedRecordPooledTrace(conn net.Conn, debug bool) ([]byte, func(), error) {
	if conn == nil {
		return nil, nil, net.ErrClosed
	}
	var headerStarted time.Time
	if debug {
		headerStarted = time.Now()
		fmt.Printf("[bulk-debug] at=%s record read header begin local=%v remote=%v\n", headerStarted.Format(time.RFC3339Nano), conn.LocalAddr(), conn.RemoteAddr())
		headerStarted = time.Now()
	}
	var header [bulkDedicatedRecordHeaderLen]byte
	if _, err := io.ReadFull(conn, header[:]); err != nil {
		if debug {
			fmt.Printf("[bulk-debug] at=%s record read header failed elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), time.Since(headerStarted), err)
		}
		return nil, nil, err
	}
	if string(header[:4]) != bulkDedicatedRecordMagic {
		return nil, nil, fmt.Errorf("%w: record magic=%x", errBulkFastPayloadInvalid, header[:4])
	}
	wireSize := binary.BigEndian.Uint32(header[4:8])
	if wireSize > bulkDedicatedRecordMaxBytes {
		return nil, nil, errBulkFastPayloadInvalid
	}
	size := int(wireSize)
	if debug {
		fmt.Printf("[bulk-debug] at=%s record read payload begin bytes=%d header=%s\n", time.Now().Format(time.RFC3339Nano), size, time.Since(headerStarted))
	}
	payload := getModernPSKPayloadBuffer(size)
	var readStarted time.Time
	if debug {
		readStarted = time.Now()
		fmt.Printf("[bulk-debug] at=%s record read buffer ready bytes=%d\n", readStarted.Format(time.RFC3339Nano), size)
		readStarted = time.Now()
	}
	n, err := io.ReadFull(conn, payload)
	if debug {
		fmt.Printf("[bulk-debug] at=%s record read payload end bytes=%d/%d elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), n, size, time.Since(readStarted), err)
	}
	if err != nil {
		putModernPSKPayloadBuffer(payload)
		return nil, nil, err
	}
	return payload, func() {
		putModernPSKPayloadBuffer(payload)
	}, nil
}

func (c *ClientCommon) dialDedicatedBulkConn(ctx context.Context, timeout time.Duration) (net.Conn, error) {
	source := c.clientConnectSourceSnapshot()
	if source != nil {
		if !source.supportsAdditionalConn() {
			return nil, errBulkDedicatedSingleConn
		}
		if source.network != "" && source.addr != "" {
			if timeout > 0 {
				return transport.DialTimeout(source.network, source.addr, timeout)
			}
			return transport.Dial(source.network, source.addr)
		}
		if source.canReconnect() {
			return source.dial(ctx)
		}
		return nil, errClientReconnectSourceUnavailable
	}
	conn := c.clientTransportConnSnapshot()
	if conn == nil || conn.RemoteAddr() == nil {
		return nil, errClientReconnectSourceUnavailable
	}
	if timeout > 0 {
		return transport.DialTimeout(conn.RemoteAddr().Network(), conn.RemoteAddr().String(), timeout)
	}
	return transport.Dial(conn.RemoteAddr().Network(), conn.RemoteAddr().String())
}

func (c *ClientCommon) attachDedicatedBulkSidecar(ctx context.Context, bulk *bulkHandle) error {
	if c == nil || bulk == nil || !bulk.Dedicated() || bulk.dedicatedAttachedSnapshot() {
		return nil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	route := bulk.clientSessionRouteSnapshot()
	if !route.bound() {
		route = c.clientSessionRouteSnapshot()
	}
	attachCtx, cancelAttach := context.WithCancel(ctx)
	defer cancelAttach()
	stopRoute := func() bool { return false }
	if route.transportStopCtx != nil {
		stopRoute = context.AfterFunc(route.transportStopCtx, cancelAttach)
		defer stopRoute()
	}
	checkRoute := func() error {
		if err := c.ensureClientSessionRouteSendReady(route); err != nil {
			return err
		}
		return nil
	}
	routeWaitError := func(waitErr error) error {
		if routeErr := checkRoute(); routeErr != nil {
			return routeErr
		}
		return waitErr
	}
	if err := checkRoute(); err != nil {
		return err
	}
	laneID := bulk.dedicatedLaneIDSnapshot()
	releaseActiveSlot, err := c.acquireBulkDedicatedActiveSlot(attachCtx)
	if err != nil {
		return routeWaitError(err)
	}
	needReleaseActive := true
	defer func() {
		if needReleaseActive {
			releaseActiveSlot()
		}
	}()
	if sidecar := c.clientDedicatedSidecarSnapshotForLaneAtRoute(laneID, route); sidecar != nil {
		if err := checkRoute(); err != nil {
			return err
		}
		if err := sidecar.withConn(func(conn net.Conn) error {
			return bulk.attachDedicatedConnShared(conn)
		}); err == nil {
			bulk.markDedicatedActiveReserved()
			needReleaseActive = false
			return nil
		}
	}
	_, flight, leader, beginErr := c.beginClientDedicatedSidecarAttach(laneID, route)
	if beginErr != nil {
		return beginErr
	}
	if !leader {
		if flight == nil {
			return errTransportDetached
		}
		if err := flight.wait(attachCtx); err != nil {
			return routeWaitError(err)
		}
		if err := checkRoute(); err != nil {
			return err
		}
		sidecar := c.clientDedicatedSidecarSnapshotForLaneAtRoute(laneID, route)
		if sidecar == nil {
			return errTransportDetached
		}
		if err := sidecar.withConn(func(conn net.Conn) error {
			return bulk.attachDedicatedConnShared(conn)
		}); err != nil {
			return err
		}
		bulk.markDedicatedActiveReserved()
		needReleaseActive = false
		return nil
	}
	if flight == nil {
		return errTransportDetached
	}
	var flightErr error
	defer func() {
		c.finishClientDedicatedSidecarAttach(laneID, flight, flightErr)
	}()
	releaseAttachSlot, err := c.acquireBulkDedicatedAttachSlot(attachCtx)
	if err != nil {
		flightErr = routeWaitError(err)
		return flightErr
	}
	defer releaseAttachSlot()
	if sidecar := c.clientDedicatedSidecarSnapshotForLaneAtRoute(laneID, route); sidecar != nil {
		if err := checkRoute(); err != nil {
			flightErr = err
			return err
		}
		if err := sidecar.withConn(func(conn net.Conn) error {
			return bulk.attachDedicatedConnShared(conn)
		}); err == nil {
			bulk.markDedicatedActiveReserved()
			needReleaseActive = false
			flightErr = nil
			return nil
		}
	}
	retry, backoff, dialTimeout, helloTimeout := c.bulkDedicatedAttachConfigSnapshot()
	attempts := retry + 1
	if attempts <= 0 {
		attempts = 1
	}
	var lastErr error
	for attempt := 1; attempt <= attempts; attempt++ {
		if err := checkRoute(); err != nil {
			flightErr = err
			return err
		}
		c.bulkAttachAttemptCount.Add(1)
		if attempt > 1 {
			delay := backoff * time.Duration(1<<(attempt-2))
			if delay > 3*time.Second {
				delay = 3 * time.Second
			}
			if err := waitDedicatedAttachBackoff(attachCtx, delay); err != nil {
				flightErr = routeWaitError(err)
				return flightErr
			}
		}
		dialCtx := attachCtx
		dialCancel := func() {}
		if dialTimeout > 0 {
			dialCtx, dialCancel = context.WithTimeout(attachCtx, dialTimeout)
		}
		conn, err := c.dialDedicatedBulkConn(dialCtx, dialTimeout)
		dialCancel()
		if err == nil && conn == nil {
			err = errTransportDetached
		}
		if err != nil {
			if routeErr := checkRoute(); routeErr != nil {
				flightErr = routeErr
				return routeErr
			}
			lastErr = err
			if attempt < attempts && isRetryableDedicatedAttachError(err) {
				flightErr = err
				c.bulkAttachRetryCount.Add(1)
				continue
			}
			bulk.markDedicatedAttachDegraded(string(bulkAttachErrorCodeAttachFailed))
			flightErr = err
			return err
		}
		if err := checkRoute(); err != nil {
			_ = conn.Close()
			flightErr = err
			return err
		}
		helloCtx := attachCtx
		helloCancel := func() {}
		if helloTimeout > 0 {
			helloCtx, helloCancel = context.WithTimeout(attachCtx, helloTimeout)
		}
		resp, err := c.sendDedicatedBulkAttachRequest(helloCtx, conn, bulk)
		helloCancel()
		if err != nil {
			_ = conn.Close()
			if routeErr := checkRoute(); routeErr != nil {
				flightErr = routeErr
				return routeErr
			}
			lastErr = err
			if attempt < attempts && isRetryableDedicatedAttachError(err) {
				flightErr = err
				c.bulkAttachRetryCount.Add(1)
				continue
			}
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
			bulk.markDedicatedAttachDegraded(string(bulkAttachErrorCodeAttachFailed))
			flightErr = err
			return err
		}
		if err := checkRoute(); err != nil {
			_ = conn.Close()
			flightErr = err
			return err
		}
		if !resp.Accepted {
			_ = conn.Close()
			rejectedErr := &bulkAttachError{
				Code:       bulkAttachErrorCode(resp.Code),
				Retryable:  resp.Retryable,
				Message:    resp.Error,
				FailedSeq:  resp.FailedSeq,
				FailedBulk: resp.FailedBulk,
			}
			if rejectedErr.Code == "" {
				rejectedErr.Code = bulkAttachErrorCodeAttachFailed
			}
			lastErr = rejectedErr
			bulk.setDedicatedAttachLastCode(string(rejectedErr.Code))
			if attempt < attempts && rejectedErr.Retryable {
				flightErr = rejectedErr
				c.bulkAttachRetryCount.Add(1)
				continue
			}
			bulk.markDedicatedAttachDegraded(string(rejectedErr.Code))
			flightErr = rejectedErr
			return rejectedErr
		}
		if err := checkRoute(); err != nil {
			_ = conn.Close()
			flightErr = err
			return err
		}
		sidecar := newBulkDedicatedSidecar(conn, laneID)
		activeSidecar, installed, installErr := c.installClientDedicatedSidecarAtRoute(laneID, sidecar, route)
		if installErr != nil {
			sidecar.close()
			flightErr = installErr
			return installErr
		}
		if !installed {
			sidecar.close()
			sidecar = activeSidecar
		}
		if sidecar == nil {
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
			flightErr = errTransportDetached
			return errTransportDetached
		}
		if err := sidecar.withConn(func(conn net.Conn) error {
			return bulk.attachDedicatedConnShared(conn)
		}); err != nil {
			if installed && c.clearClientDedicatedSidecar(laneID, sidecar) {
				sidecar.close()
			}
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
			if installed {
				flightErr = err
			} else {
				flightErr = nil
			}
			return err
		}
		if err := checkRoute(); err != nil {
			if installed && c.clearClientDedicatedSidecar(laneID, sidecar) {
				sidecar.close()
			}
			bulk.markReset(err)
			flightErr = err
			return err
		}
		c.bulkAttachSuccessCount.Add(1)
		if installed {
			go c.readDedicatedSidecarLoopAtRoute(sidecar, route)
		}
		bulk.markDedicatedActiveReserved()
		needReleaseActive = false
		flightErr = nil
		return nil
	}
	if lastErr != nil {
		flightErr = lastErr
		return lastErr
	}
	flightErr = errors.New("bulk attach failed")
	return flightErr
}

func (c *ClientCommon) bulkDedicatedAttachConfigSnapshot() (int, time.Duration, time.Duration, time.Duration) {
	if c == nil {
		return defaultBulkDedicatedAttachRetry, defaultBulkDedicatedAttachBackoff, defaultBulkDedicatedDialTimeout, defaultBulkDedicatedHelloTimeout
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	retry := c.bulkDedicatedAttachRetry
	if retry < 0 {
		retry = 0
	}
	backoff := c.bulkDedicatedAttachBackoff
	if backoff <= 0 {
		backoff = defaultBulkDedicatedAttachBackoff
	}
	dialTimeout := c.bulkDedicatedDialTimeout
	if dialTimeout <= 0 {
		dialTimeout = defaultBulkDedicatedDialTimeout
	}
	helloTimeout := c.bulkDedicatedHelloTimeout
	if helloTimeout <= 0 {
		helloTimeout = defaultBulkDedicatedHelloTimeout
	}
	return retry, backoff, dialTimeout, helloTimeout
}

func (c *ClientCommon) acquireBulkDedicatedAttachSlot(ctx context.Context) (func(), error) {
	sem := c.bulkDedicatedAttachSemaphoreSnapshot()
	if c == nil || sem == nil {
		return func() {}, nil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	select {
	case sem <- struct{}{}:
		return func() {
			select {
			case <-sem:
			default:
			}
		}, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (c *ClientCommon) reserveBulkDedicatedActiveSlot() bool {
	if c == nil {
		return false
	}
	limit := c.bulkDedicatedActiveLimitSnapshot()
	if limit <= 0 {
		return true
	}
	for {
		current := c.bulkDedicatedActive.Load()
		if int(current) >= limit {
			return false
		}
		if c.bulkDedicatedActive.CompareAndSwap(current, current+1) {
			return true
		}
	}
}

func (c *ClientCommon) acquireBulkDedicatedActiveSlot(ctx context.Context) (func(), error) {
	if c == nil {
		return func() {}, errBulkClientNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	for {
		if c.reserveBulkDedicatedActiveSlot() {
			return c.releaseBulkDedicatedActiveSlot, nil
		}
		waitCh := c.bulkDedicatedActiveWaitSnapshot()
		if c.reserveBulkDedicatedActiveSlot() {
			return c.releaseBulkDedicatedActiveSlot, nil
		}
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-waitCh:
		}
	}
}

func (c *ClientCommon) releaseBulkDedicatedActiveSlot() {
	if c == nil {
		return
	}
	for {
		current := c.bulkDedicatedActive.Load()
		if current <= 0 {
			return
		}
		if c.bulkDedicatedActive.CompareAndSwap(current, current-1) {
			c.notifyBulkDedicatedActiveWaiters()
			return
		}
	}
}

func (c *ClientCommon) notifyBulkDedicatedActiveWaiters() {
	if c == nil {
		return
	}
	c.mu.Lock()
	c.notifyBulkDedicatedActiveWaitersLocked()
	c.mu.Unlock()
}

func waitDedicatedAttachBackoff(ctx context.Context, delay time.Duration) error {
	if delay <= 0 {
		return nil
	}
	timer := time.NewTimer(delay)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-timer.C:
		return nil
	}
}

func isRetryableDedicatedAttachError(err error) bool {
	if err == nil {
		return false
	}
	var attachErr *bulkAttachError
	if errors.As(err, &attachErr) && attachErr != nil {
		return attachErr.Retryable
	}
	if errors.Is(err, context.Canceled) {
		return false
	}
	if errors.Is(err, context.DeadlineExceeded) {
		return true
	}
	var netErr net.Error
	if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) {
		return true
	}
	message := strings.ToLower(err.Error())
	for _, pattern := range []string{
		"timeout",
		"timed out",
		"deadline",
		"connection reset",
		"connection refused",
		"connectex",
		"broken pipe",
		"no route",
		"host unreachable",
		"transport detached",
	} {
		if strings.Contains(message, pattern) {
			return true
		}
	}
	return false
}

func (c *ClientCommon) sendDedicatedBulkAttachRequest(ctx context.Context, conn net.Conn, bulk *bulkHandle) (bulkAttachResponse, error) {
	if c == nil {
		return bulkAttachResponse{}, errBulkClientNil
	}
	if bulk == nil {
		return bulkAttachResponse{}, errBulkIDEmpty
	}
	if conn == nil {
		return bulkAttachResponse{}, errTransportDetached
	}
	if ctx == nil {
		ctx = context.Background()
	}
	defer func() {
		_ = conn.SetReadDeadline(time.Time{})
	}()
	reqPayload, err := c.sequenceEn(bulkAttachRequest{
		PeerID:      c.ensureClientPeerIdentity(),
		BulkID:      bulk.ID(),
		AttachToken: bulk.dedicatedAttachTokenSnapshot(),
	})
	if err != nil {
		return bulkAttachResponse{}, err
	}
	msg := TransferMsg{
		ID:    atomic.AddUint64(&c.msgID, 1),
		Key:   systemBulkAttachKey,
		Value: reqPayload,
		Type:  MSG_SYS_WAIT,
	}
	attachProfile := c.clientDedicatedBulkAttachTransportProtectionProfile()
	frame, err := encodeDirectSignalFrame(stario.NewQueue(), c.sequenceEn, attachProfile.msgEn, attachProfile.secretKey, msg)
	if err != nil {
		return bulkAttachResponse{}, err
	}
	if err := ctx.Err(); err != nil {
		return bulkAttachResponse{}, err
	}
	deadline := earlierWriteDeadline(
		writeDeadlineFromTimeout(c.maxWriteTimeoutSnapshot()),
		contextDeadline(ctx),
	)
	if err := withRawConnWriteLockDeadline(conn, deadline, func(conn net.Conn) error {
		return writeFullToConnUnlocked(conn, frame)
	}); err != nil {
		return bulkAttachResponse{}, err
	}
	if deadline, ok := ctx.Deadline(); ok {
		_ = conn.SetReadDeadline(deadline)
	}
	replyPayload, err := readDirectSignalFramePayload(conn)
	if err != nil {
		return bulkAttachResponse{}, err
	}
	transfer, err := decodeDirectSignalPayload(c.sequenceDe, attachProfile.msgDe, attachProfile.secretKey, replyPayload)
	if err != nil {
		return bulkAttachResponse{}, err
	}
	if transfer.Key != systemBulkAttachKey || transfer.Type != MSG_SYS_REPLY || transfer.ID != msg.ID {
		return bulkAttachResponse{}, errors.New("invalid bulk attach reply")
	}
	return decodeBulkAttachResponse(c.sequenceDe, transfer.Value)
}

func (c *ClientCommon) clientDedicatedBulkAttachTransportProtectionProfile() transportProtectionProfile {
	if c == nil {
		return transportProtectionProfile{}
	}
	if c.securityConfigured && c.securityBootstrap.msgEn != nil && c.securityBootstrap.msgDe != nil {
		return c.securityBootstrap.clone()
	}
	return c.clientTransportProtectionSnapshot()
}

func (c *ClientCommon) readDedicatedSidecarLoop(sidecar *bulkDedicatedSidecar) {
	c.readDedicatedSidecarLoopAtRoute(sidecar, c.clientSessionRouteSnapshot())
}

func (c *ClientCommon) readDedicatedSidecarLoopAtRoute(sidecar *bulkDedicatedSidecar, route clientSessionRoute) {
	if c == nil || sidecar == nil || sidecar.conn == nil {
		return
	}
	debug := c.IsDebugMode()
	routeCurrent := func() bool {
		if !c.clientSessionRouteCurrent(route) {
			return false
		}
		return route.transportStopCtx == nil || route.transportStopCtx.Err() == nil
	}
	for {
		if !routeCurrent() {
			return
		}
		payload, payloadRelease, err := readBulkDedicatedRecordPooledTrace(sidecar.conn, debug)
		if err != nil {
			c.handleClientDedicatedSidecarFailure(sidecar, err)
			return
		}
		if !routeCurrent() {
			if payloadRelease != nil {
				payloadRelease()
			}
			return
		}
		var dispatchStarted time.Time
		if debug {
			dispatchStarted = time.Now()
			fmt.Printf("[bulk-debug] at=%s side=client lane=%d sidecar decode-dispatch begin bytes=%d\n", dispatchStarted.Format(time.RFC3339Nano), sidecar.laneID, len(payload))
		}
		profile := c.clientTransportProtectionSnapshot()
		plain, plainRelease, err := decryptTransportPayloadCodecPooled(profile.mode, profile.runtime, profile.msgDe, profile.secretKey, payload, payloadRelease)
		if err != nil {
			c.handleClientDedicatedSidecarFailure(sidecar, err)
			return
		}
		owner := newBulkReadPayloadOwner(plainRelease)
		if !routeCurrent() {
			owner.done()
			return
		}
		runtime := c.getBulkRuntime()
		if runtime == nil {
			owner.done()
			continue
		}
		var (
			currentDataID uint64
			currentBulk   *bulkHandle
			skipDataID    bool
			staleRoute    bool
		)
		err = walkDedicatedBulkInboundPayload(plain, func(dataID uint64, item bulkDedicatedBatchItem) error {
			if !routeCurrent() {
				staleRoute = true
				currentBulk = nil
				skipDataID = true
				return nil
			}
			if dataID != currentDataID {
				currentDataID = dataID
				currentBulk = nil
				skipDataID = false
				bulk, ok := runtime.lookupInboundFrame(clientFileScope(), dataID)
				if !ok {
					c.bestEffortRejectInboundBulkDataAtRoute(route, "", dataID, errBulkNotFound.Error())
					skipDataID = true
					return nil
				}
				if !bulk.acceptsClientSessionRoute(route) {
					detachErr := transportDetachedSessionEpochError()
					bulk.markReset(detachErr)
					c.bestEffortRejectInboundBulkDataAtRoute(route, bulk.ID(), dataID, detachErr.Error())
					skipDataID = true
					return nil
				}
				bulk.markDedicatedDataStarted()
				currentBulk = bulk
			}
			if skipDataID || currentBulk == nil {
				return nil
			}
			if !routeCurrent() {
				staleRoute = true
				currentBulk = nil
				skipDataID = true
				return nil
			}
			if !currentBulk.acceptsClientSessionRoute(route) {
				detachErr := transportDetachedSessionEpochError()
				currentBulk.markReset(detachErr)
				c.bestEffortRejectInboundBulkDataAtRoute(route, currentBulk.ID(), dataID, detachErr.Error())
				currentBulk = nil
				skipDataID = true
				return nil
			}
			var release func()
			if item.Type == bulkFastPayloadTypeData {
				release = owner.retainChunk()
			}
			dispatchErr := dispatchDedicatedBulkInboundItemWithRelease(currentBulk, item, release)
			if dispatchErr != nil {
				if !errors.Is(dispatchErr, io.EOF) {
					_ = c.sendDedicatedBulkReset(context.Background(), currentBulk, dispatchErr.Error())
					currentBulk.markReset(dispatchErr)
				}
				currentBulk = nil
				skipDataID = true
				return nil
			}
			if currentBulk.Context().Err() != nil {
				currentBulk = nil
				skipDataID = true
			}
			return nil
		})
		if err != nil {
			if plainRelease != nil {
				plainRelease()
			}
			c.handleClientDedicatedSidecarFailure(sidecar, err)
			return
		}
		owner.done()
		if debug {
			fmt.Printf("[bulk-debug] at=%s side=client lane=%d sidecar decode-dispatch end elapsed=%s\n", time.Now().Format(time.RFC3339Nano), sidecar.laneID, time.Since(dispatchStarted))
		}
		if staleRoute {
			return
		}
	}
}

func (s *ServerCommon) handleBulkAttachSystemMessage(message Message) bool {
	if message.Key != systemBulkAttachKey {
		return false
	}
	current := messageLogicalConnSnapshot(&message)
	currentTransport := message.TransportConn
	if currentTransport == nil && current != nil {
		currentTransport = current.CurrentTransportConn()
	}
	if currentTransport != nil && !currentTransport.IsCurrent() {
		if current != nil {
			_ = s.replyDedicatedBulkAttach(current, message, toBulkAttachResponseError(newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, transportDetachedErrorForTransport(currentTransport).Error()), ""))
		}
		return true
	}
	var (
		req     bulkAttachRequest
		logical *LogicalConn
		bulk    *bulkHandle
		err     error
	)
	req, err = decodeBulkAttachRequest(s.sequenceDe, message.Value)
	if err == nil {
		logical, bulk, err = s.resolveInboundDedicatedBulk(current, req)
	}
	if err != nil {
		if current != nil {
			_ = s.replyDedicatedBulkAttach(current, message, toBulkAttachResponseError(err, req.BulkID))
		}
		return true
	}
	if current != nil {
		if attachErr := s.finishInboundDedicatedBulkAttach(current, logical, bulk, message); attachErr != nil {
			bulk.markReset(attachErr)
		}
	}
	return true
}

func (s *ServerCommon) resolveInboundDedicatedBulk(current *LogicalConn, req bulkAttachRequest) (*LogicalConn, *bulkHandle, error) {
	if s == nil {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodeServerUnavailable, true, errBulkServerNil.Error())
	}
	if current == nil {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodeInvalidRequest, false, errBulkLogicalConnNil.Error())
	}
	if req.PeerID == "" || req.BulkID == "" || req.AttachToken == "" {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodeInvalidRequest, false, errBulkIDEmpty.Error())
	}
	logical := s.GetLogicalConn(req.PeerID)
	if logical == nil {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodePeerNotFound, true, errBulkLogicalConnNil.Error())
	}
	runtime := s.getBulkRuntime()
	if runtime == nil {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodeServerUnavailable, true, errBulkRuntimeNil.Error())
	}
	bulk, ok := runtime.lookup(serverFileScope(logical), req.BulkID)
	if !ok {
		return nil, nil, &bulkAttachError{
			Code:       bulkAttachErrorCodeBulkNotFound,
			Retryable:  false,
			Message:    errBulkNotFound.Error(),
			FailedBulk: req.BulkID,
		}
	}
	bulk.markDedicatedAttachAttempt()
	if bulkTransport := bulk.TransportConn(); bulkTransport != nil && !bulkTransport.IsCurrent() {
		return nil, nil, newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, transportDetachedErrorForTransport(bulkTransport).Error())
	}
	if !bulk.Dedicated() {
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeBulkNotDedicated))
		return nil, nil, &bulkAttachError{
			Code:       bulkAttachErrorCodeBulkNotDedicated,
			Retryable:  false,
			Message:    "bulk is not dedicated",
			FailedBulk: req.BulkID,
		}
	}
	if bulk.dedicatedAttachTokenSnapshot() != req.AttachToken {
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeTokenMismatch))
		return nil, nil, &bulkAttachError{
			Code:       bulkAttachErrorCodeTokenMismatch,
			Retryable:  false,
			Message:    "bulk attach token mismatch",
			FailedBulk: req.BulkID,
		}
	}
	switch bulk.dedicatedAttachStateSnapshot() {
	case bulkDedicatedAttachStateClosed:
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return nil, nil, &bulkAttachError{
			Code:       bulkAttachErrorCodeAttachFailed,
			Retryable:  false,
			Message:    "bulk dedicated attach closed",
			FailedBulk: req.BulkID,
		}
	case bulkDedicatedAttachStateDegraded:
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return nil, nil, &bulkAttachError{
			Code:       bulkAttachErrorCodeAttachFailed,
			Retryable:  true,
			Message:    "bulk dedicated attach degraded",
			FailedBulk: req.BulkID,
		}
	case bulkDedicatedAttachStateAttached:
		if bulk.dedicatedDataStartedSnapshot() {
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAlreadyAttached))
			return nil, nil, &bulkAttachError{
				Code:       bulkAttachErrorCodeAlreadyAttached,
				Retryable:  false,
				Message:    "bulk dedicated already attached",
				FailedBulk: req.BulkID,
			}
		}
	}
	return logical, bulk, nil
}

func (s *ServerCommon) finishInboundDedicatedBulkAttach(current *LogicalConn, logical *LogicalConn, bulk *bulkHandle, message Message) error {
	if current == nil || logical == nil || bulk == nil {
		return newBulkAttachError(bulkAttachErrorCodeInvalidRequest, false, errBulkLogicalConnNil.Error())
	}
	// Keep the target logical transport generation stable while the attach
	// reply, sidecar publication, runtime rebinding and accept dispatch are
	// committed. Reattach cleanup takes the same lock before retiring the old
	// generation.
	logical.transportLifecycleMu.Lock()
	defer logical.transportLifecycleMu.Unlock()
	scope := serverFileScope(logical)
	laneID := bulk.dedicatedLaneIDSnapshot()
	currentTransport := message.TransportConn
	if currentTransport == nil {
		currentTransport = current.CurrentTransportConn()
	}
	if currentTransport != nil && !currentTransport.IsCurrent() {
		return newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, transportDetachedErrorForTransport(currentTransport).Error())
	}
	if bulkTransport := bulk.TransportConn(); bulkTransport != nil && !bulkTransport.IsCurrent() {
		return newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, transportDetachedErrorForTransport(bulkTransport).Error())
	}
	conn, err := current.detachTransportForTransfer()
	if err != nil {
		return newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, err.Error())
	}
	stopCurrent := func(reason string, err error) {
		current.markSessionStopped(reason, err)
		s.removeLogical(current)
	}
	fail := func(reason string, err error) error {
		if conn != nil {
			_ = conn.Close()
		}
		bulk.markDedicatedAttachDegraded(string(bulkAttachErrorCodeAttachFailed))
		stopCurrent(reason, err)
		return newBulkAttachError(bulkAttachErrorCodeAttachFailed, true, err.Error())
	}
	if bulk.dedicatedAttachedSnapshot() {
		if bulk.dedicatedDataStartedSnapshot() {
			rejected := &bulkAttachError{
				Code:       bulkAttachErrorCodeAlreadyAttached,
				Retryable:  false,
				Message:    "bulk dedicated already attached",
				FailedBulk: bulk.ID(),
			}
			_ = s.replyDedicatedBulkAttachDetached(current, conn, message, toBulkAttachResponseError(rejected, bulk.ID()))
			_ = conn.Close()
			stopCurrent("bulk dedicated attach duplicate rejected", nil)
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAlreadyAttached))
			return nil
		}
	}
	sidecar := newBulkDedicatedSidecar(conn, laneID)
	if sidecar == nil {
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return fail("bulk dedicated attach failed", errTransportDetached)
	}
	var (
		oldConn   net.Conn
		oldSender *bulkDedicatedSender
	)
	if bulk.dedicatedAttachedSnapshot() {
		var replaceErr error
		oldConn, oldSender, replaceErr = bulk.replaceDedicatedConnShared(conn)
		if replaceErr != nil {
			sidecar.close()
			bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
			return fail("bulk dedicated reattach failed", replaceErr)
		}
	} else if err := bulk.attachDedicatedConnShared(conn); err != nil {
		sidecar.close()
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return fail("bulk dedicated attach failed", err)
	}
	if bulkTransport := bulk.TransportConn(); bulkTransport != nil && !bulkTransport.IsCurrent() {
		sidecar.close()
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return fail("bulk dedicated attach transport replaced", transportDetachedErrorForTransport(bulkTransport))
	}
	if err := s.replyDedicatedBulkAttachDetached(current, conn, message, bulkAttachResponse{Accepted: true}); err != nil {
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		sidecar.close()
		if runtime := s.getBulkRuntime(); runtime != nil {
			runtime.resetDedicatedByConn(scope, conn, transportDetachedError("bulk dedicated attach reply failed", err))
		}
		stopCurrent("bulk dedicated attach reply failed", err)
		return nil
	}
	if bulkTransport := bulk.TransportConn(); bulkTransport != nil && !bulkTransport.IsCurrent() {
		sidecar.close()
		bulk.setDedicatedAttachLastCode(string(bulkAttachErrorCodeAttachFailed))
		return fail("bulk dedicated attach transport replaced", transportDetachedErrorForTransport(bulkTransport))
	}
	oldSidecar := s.installServerDedicatedSidecar(logical, laneID, sidecar)
	if runtime := s.getBulkRuntime(); runtime != nil {
		runtime.attachSharedDedicatedConn(scope, laneID, conn)
	}
	if oldSender != nil {
		oldSender.stop()
	}
	if oldConn != nil {
		_ = oldConn.Close()
	}
	if oldSidecar != nil && oldSidecar != sidecar {
		oldSidecar.close()
	}
	go s.readDedicatedSidecarLoop(logical, sidecar)
	s.startServerBulkAcceptDispatch(bulk, logical, bulk.TransportConn())
	if runtime := s.getBulkRuntime(); runtime != nil {
		s.dispatchPendingServerBulkAccepts(scope, conn, bulk, logical)
	}
	stopCurrent("bulk dedicated attach", nil)
	return nil
}

func (s *ServerCommon) dispatchPendingServerBulkAccepts(scope string, conn net.Conn, current *bulkHandle, logical *LogicalConn) {
	if s == nil || conn == nil {
		return
	}
	runtime := s.getBulkRuntime()
	if runtime == nil {
		return
	}
	for _, pending := range runtime.dedicatedBulksForConn(scope, conn) {
		if pending == nil || pending == current {
			continue
		}
		s.startServerBulkAcceptDispatch(pending, logical, pending.TransportConn())
	}
}

func (s *ServerCommon) replyDedicatedBulkAttachDetached(client *LogicalConn, conn net.Conn, message Message, resp bulkAttachResponse) error {
	if s == nil || client == nil {
		return errBulkServerNil
	}
	if conn == nil {
		return net.ErrClosed
	}
	msgEn := client.msgEnSnapshot()
	if msgEn == nil {
		return errTransportPayloadEncryptFailed
	}
	encoded, err := s.sequenceEn(resp)
	if err != nil {
		return err
	}
	reply := TransferMsg{
		ID:    message.ID,
		Key:   systemBulkAttachKey,
		Value: encoded,
		Type:  MSG_SYS_REPLY,
	}
	frame, err := encodeDirectSignalFrame(stario.NewQueue(), s.sequenceEn, msgEn, client.secretKeySnapshot(), reply)
	if err != nil {
		return err
	}
	deadline := earlierWriteDeadline(
		writeDeadlineFromTimeout(defaultBulkDedicatedHelloTimeout),
		writeDeadlineFromTimeout(client.maxWriteTimeoutSnapshot()),
	)
	return withRawConnWriteLockDeadline(conn, deadline, func(conn net.Conn) error {
		return writeFullToConnUnlocked(conn, frame)
	})
}

func (s *ServerCommon) replyDedicatedBulkAttach(client *LogicalConn, message Message, resp bulkAttachResponse) error {
	if s == nil || client == nil {
		return errBulkServerNil
	}
	encoded, err := s.sequenceEn(resp)
	if err != nil {
		return err
	}
	reply := TransferMsg{
		ID:    message.ID,
		Key:   systemBulkAttachKey,
		Value: encoded,
		Type:  MSG_SYS_REPLY,
	}
	if message.inboundConn != nil {
		return s.sendTransferInbound(client, messageTransportConnSnapshot(&message), message.inboundConn, messageInboundTransportProtectionSnapshot(&message), reply)
	}
	_, err = s.sendLogical(client, reply)
	return err
}

func (s *ServerCommon) readDedicatedSidecarLoop(logical *LogicalConn, sidecar *bulkDedicatedSidecar) {
	if s == nil || logical == nil || sidecar == nil || sidecar.conn == nil {
		return
	}
	debug := s.IsDebugMode()
	scope := serverFileScope(logical)
	sidecarCurrent := func() bool {
		return s.serverDedicatedSidecarCurrent(logical, sidecar)
	}
	for {
		if !sidecarCurrent() {
			if debug {
				fmt.Printf("[bulk-debug] at=%s side=server lane=%d sidecar stopped reason=stale-connection\n", time.Now().Format(time.RFC3339Nano), sidecar.laneID)
			}
			return
		}
		var readStarted time.Time
		if debug {
			readStarted = time.Now()
		}
		payload, payloadRelease, err := readBulkDedicatedRecordPooled(sidecar.conn)
		if debug {
			fmt.Printf("[bulk-debug] at=%s side=server lane=%d sidecar record read bytes=%d elapsed=%s error=%v\n", time.Now().Format(time.RFC3339Nano), sidecar.laneID, len(payload), time.Since(readStarted), err)
		}
		if err != nil {
			if !sidecarCurrent() {
				return
			}
			s.handleServerDedicatedSidecarFailure(logical, sidecar, err)
			return
		}
		if !sidecarCurrent() {
			if payloadRelease != nil {
				payloadRelease()
			}
			return
		}
		plain, plainRelease, err := decryptTransportPayloadCodecPooled(logical.protectionModeSnapshot(), logical.modernPSKRuntimeSnapshot(), logical.msgDeSnapshot(), logical.secretKeySnapshot(), payload, payloadRelease)
		if err != nil {
			if !sidecarCurrent() {
				return
			}
			s.handleServerDedicatedSidecarFailure(logical, sidecar, err)
			return
		}
		owner := newBulkReadPayloadOwner(plainRelease)
		if !sidecarCurrent() {
			owner.done()
			return
		}
		runtime := s.getBulkRuntime()
		if runtime == nil {
			owner.done()
			continue
		}
		var (
			currentDataID uint64
			currentBulk   *bulkHandle
			skipDataID    bool
			staleSidecar  bool
		)
		err = walkDedicatedBulkInboundPayload(plain, func(dataID uint64, item bulkDedicatedBatchItem) error {
			if debug && item.Type == bulkFastPayloadTypeRelease {
				fmt.Printf("[bulk-debug] at=%s side=server lane=%d data=%d sidecar release received payload=%d\n", time.Now().Format(time.RFC3339Nano), sidecar.laneID, dataID, len(item.Payload))
			}
			if !sidecarCurrent() {
				staleSidecar = true
				currentBulk = nil
				skipDataID = true
				return nil
			}
			if dataID != currentDataID {
				currentDataID = dataID
				currentBulk = nil
				skipDataID = false
				bulk, ok := runtime.lookupInboundFrame(scope, dataID)
				if !ok {
					if debug {
						fmt.Printf("[bulk-debug] at=%s side=server id=- data=%d age=- sidecar inbound lookup failed type=%d\n", time.Now().Format(time.RFC3339Nano), dataID, item.Type)
					}
					if sidecarCurrent() {
						s.bestEffortRejectInboundDedicatedData(logical, sidecar.conn, dataID, errBulkNotFound.Error())
					} else {
						staleSidecar = true
					}
					skipDataID = true
					return nil
				}
				if !bulkDedicatedSidecarConnCurrent(bulk, sidecar) {
					if debug {
						bulk.debugf("sidecar reject lane=%d type=%d reason=connection-mismatch", sidecar.laneID, item.Type)
					}
					skipDataID = true
					return nil
				}
				bulk.markDedicatedDataStarted()
				currentBulk = bulk
			}
			if skipDataID || currentBulk == nil {
				return nil
			}
			if !sidecarCurrent() || !bulkDedicatedSidecarConnCurrent(currentBulk, sidecar) {
				if !sidecarCurrent() {
					staleSidecar = true
				}
				currentBulk = nil
				skipDataID = true
				return nil
			}

			var release func()
			if item.Type == bulkFastPayloadTypeData {
				release = owner.retainChunk()
			}
			if !sidecarCurrent() || !bulkDedicatedSidecarConnCurrent(currentBulk, sidecar) {
				if release != nil {
					release()
				}
				if !sidecarCurrent() {
					staleSidecar = true
				}
				currentBulk = nil
				skipDataID = true
				return nil
			}
			dispatchErr := dispatchDedicatedBulkInboundItemWithRelease(currentBulk, item, release)

			if dispatchErr != nil {
				if debug {
					currentBulk.debugf("sidecar dispatch failed lane=%d type=%d error=%v", sidecar.laneID, item.Type, dispatchErr)
				}
				if !sidecarCurrent() {
					staleSidecar = true
					currentBulk = nil
					skipDataID = true
					return nil
				}
				if !errors.Is(dispatchErr, io.EOF) {
					_ = s.sendDedicatedBulkReset(context.Background(), logical, currentBulk, dispatchErr.Error())
					currentBulk.markReset(dispatchErr)
				}
				currentBulk = nil
				skipDataID = true
				return nil
			}
			if currentBulk.Context().Err() != nil {
				currentBulk = nil
				skipDataID = true
			}
			return nil
		})
		if err != nil {
			owner.done()
			if !sidecarCurrent() {
				return
			}
			s.handleServerDedicatedSidecarFailure(logical, sidecar, err)
			return
		}
		owner.done()
		if staleSidecar {
			return
		}
	}
}

func (s *ServerCommon) bestEffortRejectInboundDedicatedData(logical *LogicalConn, conn net.Conn, dataID uint64, message string) {
	if s == nil || logical == nil || conn == nil || dataID == 0 {
		return
	}
	frame, err := s.encodeDedicatedBulkBatchPayload(logical, dataID, []bulkDedicatedSendRequest{{
		Type:    bulkFastPayloadTypeReset,
		Payload: []byte(message),
	}})
	if err != nil {
		return
	}
	_ = writeBulkDedicatedRecordWithDeadline(conn, frame, writeDeadlineFromTimeout(logical.maxWriteTimeoutSnapshot()))
}

func handleDedicatedBulkReadError(bulk *bulkHandle, err error) {
	if bulk == nil {
		return
	}
	if bulk.Context().Err() != nil || bulk.remoteClosedSnapshot() {
		return
	}
	message := strings.ToLower(err.Error())
	if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || strings.Contains(message, "use of closed network connection") {
		if bulk.Dedicated() || bulk.localClosedSnapshot() {
			bulk.markRemoteClosed()
			return
		}
	}
	bulk.markReset(transportDetachedError("dedicated bulk read error", err))
}

func (c *ClientCommon) dedicatedBulkSender(bulk *bulkHandle) (*bulkDedicatedSender, error) {
	if c == nil || bulk == nil {
		return nil, errBulkClientNil
	}
	if sender := bulk.dedicatedSenderSnapshot(); sender != nil {
		return sender, nil
	}
	conn := bulk.dedicatedConnSnapshot()
	if conn == nil {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	sender := newBulkDedicatedSender(conn, bulk.dataIDSnapshot(), c.encryptTransportPayload, func(items []bulkDedicatedSendRequest) ([]byte, error) {
		return c.encodeDedicatedBulkBatchPayload(bulk.dataIDSnapshot(), items)
	}, func(err error) {
		if bulk.canIgnoreDedicatedCloseSendError(err) {
			return
		}
		bulk.markReset(err)
	})
	actual := bulk.installDedicatedSender(sender)
	if actual != sender {
		sender.stop()
	}
	if actual == nil {
		return nil, io.ErrClosedPipe
	}
	return actual, nil
}

func (c *ClientCommon) dedicatedBulkLaneSender(bulk *bulkHandle) (*bulkDedicatedLaneSender, error) {
	if c == nil || bulk == nil {
		return nil, errBulkClientNil
	}
	route := bulk.clientSessionRouteSnapshot()
	sidecar := c.clientDedicatedSidecarSnapshotForLaneAtRoute(bulk.dedicatedLaneIDSnapshot(), route)
	conn := bulk.dedicatedConnSnapshot()
	if sidecar == nil || sidecar.conn == nil || conn == nil || sidecar.conn != conn {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	sender := sidecar.laneSenderWithFactory(func(conn net.Conn) *bulkDedicatedLaneSender {
		profile := c.clientTransportProtectionSnapshot()
		laneRuntime := profile.runtime
		if forked, err := forkDedicatedLaneModernPSKRuntime(laneRuntime); err == nil && forked != nil {
			laneRuntime = forked
		}
		return newBulkDedicatedLaneSender(conn, func(batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
			return c.encodeDedicatedBulkBatchesPayloadPooledWithRuntime(laneRuntime, batches)
		}, func(err error) {
			c.handleClientDedicatedSidecarFailure(sidecar, err)
		}, c.IsDebugMode())
	})
	if sender == nil {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	return sender, nil
}

func (c *ClientCommon) sendDedicatedBulkData(ctx context.Context, bulk *bulkHandle, chunk []byte) error {
	if c == nil || bulk == nil {
		return errBulkClientNil
	}
	sender, err := c.dedicatedBulkLaneSender(bulk)
	if err != nil {
		return err
	}
	return sender.submitData(ctx, bulk.dataIDSnapshot(), bulk.nextOutboundDataSeq(), chunk)
}

func (c *ClientCommon) sendDedicatedBulkWrite(ctx context.Context, bulk *bulkHandle, startSeq uint64, payload []byte, payloadOwned bool) (int, error) {
	if c == nil || bulk == nil {
		return 0, errBulkClientNil
	}
	sender, err := c.dedicatedBulkLaneSender(bulk)
	if err != nil {
		return 0, err
	}
	return sender.submitWrite(ctx, bulk.dataIDSnapshot(), startSeq, payload, bulk.chunkSize, payloadOwned)
}

func (c *ClientCommon) sendDedicatedBulkClose(ctx context.Context, bulk *bulkHandle, full bool) error {
	if c == nil || bulk == nil {
		return errBulkClientNil
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	flags := uint8(0)
	if full {
		flags = bulkFastPayloadFlagFullClose
	}
	sender, err := c.dedicatedBulkLaneSender(bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeClose, flags, 0, nil)
}

func (c *ClientCommon) sendDedicatedBulkReset(ctx context.Context, bulk *bulkHandle, message string) error {
	if c == nil || bulk == nil {
		return errBulkClientNil
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	sender, err := c.dedicatedBulkLaneSender(bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeReset, 0, 0, []byte(message))
}

func (c *ClientCommon) sendDedicatedBulkRelease(ctx context.Context, bulk *bulkHandle, bytes int64, chunks int) error {
	if c == nil || bulk == nil {
		return errBulkClientNil
	}
	payload, err := encodeBulkDedicatedReleasePayload(bytes, chunks)
	if err != nil {
		return err
	}
	if err := bulk.waitDedicatedReady(ctx); err != nil {
		return err
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	sender, err := c.dedicatedBulkLaneSender(bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeRelease, 0, 0, payload)
}

func (s *ServerCommon) dedicatedBulkSender(logical *LogicalConn, bulk *bulkHandle) (*bulkDedicatedSender, error) {
	if s == nil || bulk == nil {
		return nil, errBulkServerNil
	}
	if logical == nil {
		logical = bulk.LogicalConn()
	}
	if logical == nil {
		return nil, errBulkLogicalConnNil
	}
	if sender := bulk.dedicatedSenderSnapshot(); sender != nil {
		return sender, nil
	}
	conn := bulk.dedicatedConnSnapshot()
	if conn == nil {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	sender := newBulkDedicatedSender(conn, bulk.dataIDSnapshot(), func(plain []byte) ([]byte, error) {
		return s.encryptTransportPayloadLogical(logical, plain)
	}, func(items []bulkDedicatedSendRequest) ([]byte, error) {
		return s.encodeDedicatedBulkBatchPayload(logical, bulk.dataIDSnapshot(), items)
	}, func(err error) {
		if bulk.canIgnoreDedicatedCloseSendError(err) {
			return
		}
		bulk.markReset(err)
	})
	actual := bulk.installDedicatedSender(sender)
	if actual != sender {
		sender.stop()
	}
	if actual == nil {
		return nil, io.ErrClosedPipe
	}
	return actual, nil
}

func (s *ServerCommon) dedicatedBulkLaneSender(logical *LogicalConn, bulk *bulkHandle) (*bulkDedicatedLaneSender, error) {
	if s == nil || bulk == nil {
		return nil, errBulkServerNil
	}
	if logical == nil {
		logical = bulk.LogicalConn()
	}
	if logical == nil {
		return nil, errBulkLogicalConnNil
	}
	sidecar := s.serverDedicatedSidecarSnapshotForLane(logical, bulk.dedicatedLaneIDSnapshot())
	conn := bulk.dedicatedConnSnapshot()
	if sidecar == nil || sidecar.conn == nil || conn == nil || sidecar.conn != conn {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	sender := sidecar.laneSenderWithFactory(func(conn net.Conn) *bulkDedicatedLaneSender {
		laneRuntime := logical.modernPSKRuntimeSnapshot()
		if forked, err := forkDedicatedLaneModernPSKRuntime(laneRuntime); err == nil && forked != nil {
			laneRuntime = forked
		}
		return newBulkDedicatedLaneSender(conn, func(batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
			return s.encodeDedicatedBulkBatchesPayloadPooledWithRuntime(logical, laneRuntime, batches)
		}, func(err error) {
			s.handleServerDedicatedSidecarFailure(logical, sidecar, err)
		}, s.IsDebugMode())
	})
	if sender == nil {
		return nil, transportDetachedError("dedicated bulk sidecar not attached", nil)
	}
	return sender, nil
}

func (s *ServerCommon) sendDedicatedBulkData(ctx context.Context, logical *LogicalConn, bulk *bulkHandle, chunk []byte) error {
	if s == nil || bulk == nil {
		return errBulkServerNil
	}
	sender, err := s.dedicatedBulkLaneSender(logical, bulk)
	if err != nil {
		return err
	}
	return sender.submitData(ctx, bulk.dataIDSnapshot(), bulk.nextOutboundDataSeq(), chunk)
}

func (s *ServerCommon) sendDedicatedBulkWrite(ctx context.Context, logical *LogicalConn, bulk *bulkHandle, startSeq uint64, payload []byte, payloadOwned bool) (int, error) {
	if s == nil || bulk == nil {
		return 0, errBulkServerNil
	}
	sender, err := s.dedicatedBulkLaneSender(logical, bulk)
	if err != nil {
		return 0, err
	}
	return sender.submitWrite(ctx, bulk.dataIDSnapshot(), startSeq, payload, bulk.chunkSize, payloadOwned)
}

func (s *ServerCommon) sendDedicatedBulkClose(ctx context.Context, logical *LogicalConn, bulk *bulkHandle, full bool) error {
	if s == nil || bulk == nil {
		return errBulkServerNil
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	flags := uint8(0)
	if full {
		flags = bulkFastPayloadFlagFullClose
	}
	sender, err := s.dedicatedBulkLaneSender(logical, bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeClose, flags, 0, nil)
}

func (s *ServerCommon) sendDedicatedBulkReset(ctx context.Context, logical *LogicalConn, bulk *bulkHandle, message string) error {
	if s == nil || bulk == nil {
		return errBulkServerNil
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	sender, err := s.dedicatedBulkLaneSender(logical, bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeReset, 0, 0, []byte(message))
}

func (s *ServerCommon) sendDedicatedBulkRelease(ctx context.Context, logical *LogicalConn, bulk *bulkHandle, bytes int64, chunks int) error {
	if s == nil || bulk == nil {
		return errBulkServerNil
	}
	payload, err := encodeBulkDedicatedReleasePayload(bytes, chunks)
	if err != nil {
		return err
	}
	if err := bulk.waitDedicatedReady(ctx); err != nil {
		return err
	}
	sendCtx, cancel, err := bulkWriteContext(ctx, bulk.writeTimeout)
	if err != nil {
		return err
	}
	defer cancel()
	sender, err := s.dedicatedBulkLaneSender(logical, bulk)
	if err != nil {
		return err
	}
	return sender.submitControl(sendCtx, bulk.dataIDSnapshot(), bulkFastPayloadTypeRelease, 0, 0, payload)
}

func (c *ClientCommon) encodeDedicatedBulkBatchPayload(dataID uint64, items []bulkDedicatedSendRequest) ([]byte, error) {
	if c == nil {
		return nil, errBulkClientNil
	}
	profile := c.clientTransportProtectionSnapshot()
	if runtime := profile.runtime; runtime != nil {
		plainLen, err := bulkDedicatedBatchPlainLenChecked(items)
		if err != nil {
			return nil, err
		}
		return runtime.sealFilledPayload(plainLen, func(dst []byte) error {
			return writeBulkDedicatedBatchPlain(dst, dataID, items)
		})
	}
	if profile.fastPlainEncode != nil {
		return encodeBulkDedicatedBatchPayloadFast(profile.fastPlainEncode, profile.secretKey, dataID, items)
	}
	plain, err := encodeBulkDedicatedBatchPlain(dataID, items)
	if err != nil {
		return nil, err
	}
	return c.encryptTransportPayload(plain)
}

func forkDedicatedLaneModernPSKRuntime(base *modernPSKCodecRuntime) (*modernPSKCodecRuntime, error) {
	if base == nil {
		return nil, nil
	}
	return base.fork()
}

func (c *ClientCommon) encodeDedicatedBulkBatchesPayloadPooledWithRuntime(runtime *modernPSKCodecRuntime, batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
	if c == nil {
		return nil, nil, errBulkClientNil
	}
	if len(batches) == 0 {
		return nil, nil, errBulkFastPayloadInvalid
	}
	if runtime != nil {
		plainLen, err := bulkDedicatedBatchesPlainLenChecked(batches)
		if err != nil {
			return nil, nil, err
		}
		return runtime.sealFilledPayloadPooled(plainLen, func(dst []byte) error {
			return writeBulkDedicatedBatchesPlain(dst, batches)
		})
	}
	profile := c.clientTransportProtectionSnapshot()
	if profile.fastPlainEncode != nil {
		payload, err := encodeBulkDedicatedBatchesPayloadFast(profile.fastPlainEncode, profile.secretKey, batches)
		return payload, nil, err
	}
	plain, err := encodeBulkDedicatedBatchesPlain(batches)
	if err != nil {
		return nil, nil, err
	}
	payload, err := c.encryptTransportPayload(plain)
	return payload, nil, err
}

func (c *ClientCommon) encodeDedicatedBulkBatchesPayloadPooled(batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
	return c.encodeDedicatedBulkBatchesPayloadPooledWithRuntime(c.clientTransportProtectionSnapshot().runtime, batches)
}

func (c *ClientCommon) encodeDedicatedBulkBatchPayloadPooled(dataID uint64, items []bulkDedicatedSendRequest) ([]byte, func(), error) {
	return c.encodeDedicatedBulkBatchesPayloadPooled([]bulkDedicatedOutboundBatch{{
		DataID: dataID,
		Items:  items,
	}})
}

func (s *ServerCommon) encodeDedicatedBulkBatchPayload(logical *LogicalConn, dataID uint64, items []bulkDedicatedSendRequest) ([]byte, error) {
	if s == nil {
		return nil, errBulkServerNil
	}
	if logical == nil {
		return nil, errBulkLogicalConnNil
	}
	if runtime := logical.modernPSKRuntimeSnapshot(); runtime != nil {
		plainLen, err := bulkDedicatedBatchPlainLenChecked(items)
		if err != nil {
			return nil, err
		}
		return runtime.sealFilledPayload(plainLen, func(dst []byte) error {
			return writeBulkDedicatedBatchPlain(dst, dataID, items)
		})
	}
	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
		return encodeBulkDedicatedBatchPayloadFast(fastPlainEncode, logical.secretKeySnapshot(), dataID, items)
	}
	plain, err := encodeBulkDedicatedBatchPlain(dataID, items)
	if err != nil {
		return nil, err
	}
	return s.encryptTransportPayloadLogical(logical, plain)
}

func (s *ServerCommon) encodeDedicatedBulkBatchesPayloadPooledWithRuntime(logical *LogicalConn, runtime *modernPSKCodecRuntime, batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
	if s == nil {
		return nil, nil, errBulkServerNil
	}
	if logical == nil {
		return nil, nil, errBulkLogicalConnNil
	}
	if len(batches) == 0 {
		return nil, nil, errBulkFastPayloadInvalid
	}
	if runtime != nil {
		plainLen, err := bulkDedicatedBatchesPlainLenChecked(batches)
		if err != nil {
			return nil, nil, err
		}
		return runtime.sealFilledPayloadPooled(plainLen, func(dst []byte) error {
			return writeBulkDedicatedBatchesPlain(dst, batches)
		})
	}
	if fastPlainEncode := logical.fastPlainEncodeSnapshot(); fastPlainEncode != nil {
		payload, err := encodeBulkDedicatedBatchesPayloadFast(fastPlainEncode, logical.secretKeySnapshot(), batches)
		return payload, nil, err
	}
	plain, err := encodeBulkDedicatedBatchesPlain(batches)
	if err != nil {
		return nil, nil, err
	}
	payload, err := s.encryptTransportPayloadLogical(logical, plain)
	return payload, nil, err
}

func (s *ServerCommon) encodeDedicatedBulkBatchesPayloadPooled(logical *LogicalConn, batches []bulkDedicatedOutboundBatch) ([]byte, func(), error) {
	return s.encodeDedicatedBulkBatchesPayloadPooledWithRuntime(logical, logical.modernPSKRuntimeSnapshot(), batches)
}

func (s *ServerCommon) encodeDedicatedBulkBatchPayloadPooled(logical *LogicalConn, dataID uint64, items []bulkDedicatedSendRequest) ([]byte, func(), error) {
	return s.encodeDedicatedBulkBatchesPayloadPooled(logical, []bulkDedicatedOutboundBatch{{
		DataID: dataID,
		Items:  items,
	}})
}