notify/record_stream.go

package notify

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

type RecordErrorCode string

const (
	RecordErrorCodeApplyFailed RecordErrorCode = "apply_failed"
	RecordErrorCodeProtocol    RecordErrorCode = "protocol"
	RecordErrorCodeCanceled    RecordErrorCode = "canceled"
)

const (
	defaultRecordMaxBatchRecords   = 64
	defaultRecordMaxBatchBytes     = 128 * 1024
	defaultRecordMaxBatchDelay     = 2 * time.Millisecond
	defaultRecordMaxUnackedRecords = 4096
	defaultRecordMaxUnackedBytes   = 8 * 1024 * 1024
	defaultRecordInboundQueueLimit = 128
	defaultRecordAckEveryRecords   = 64
	defaultRecordAckDelay          = time.Millisecond
)

type RecordFailure struct {
	FailedSeq uint64
	Code      RecordErrorCode
	Retryable bool
	Message   string
}

func (f RecordFailure) Error() string {
	if f.FailedSeq == 0 && f.Code == "" && f.Message == "" {
		return "record stream failed"
	}
	if f.Message == "" {
		return fmt.Sprintf("record stream failed: seq=%d code=%s retryable=%t", f.FailedSeq, f.Code, f.Retryable)
	}
	return fmt.Sprintf("record stream failed: seq=%d code=%s retryable=%t msg=%s", f.FailedSeq, f.Code, f.Retryable, f.Message)
}

type RecordMessage struct {
	Seq     uint64
	Payload []byte
}

type RecordOpenOptions struct {
	Stream            StreamOpenOptions
	MaxBatchRecords   int
	MaxBatchBytes     int
	MaxBatchDelay     time.Duration
	MaxUnackedRecords int
	MaxUnackedBytes   int
	InboundQueueLimit int
	AckEveryRecords   int
	AckDelay          time.Duration
}

type RecordAcceptInfo struct {
	ID                  string
	Metadata            StreamMetadata
	LogicalConn         *LogicalConn
	TransportConn       *TransportConn
	TransportGeneration uint64
	RecordStream        RecordStream
}

type RecordStream interface {
	ID() string
	Metadata() StreamMetadata
	Context() context.Context

	ReadRecord(context.Context) (RecordMessage, error)
	WriteRecord(context.Context, []byte) (uint64, error)
	Flush(context.Context) error
	BarrierTo(context.Context, uint64) (uint64, error)
	Barrier(context.Context) (uint64, error)

	AckRecord(uint64) error
	FailRecord(uint64, RecordFailure) error

	CloseWrite() error
	Close() error
	Reset(error) error
}

type recordConfig struct {
	MaxBatchRecords   int
	MaxBatchBytes     int
	MaxBatchDelay     time.Duration
	MaxUnackedRecords int
	MaxUnackedBytes   int
	InboundQueueLimit int
	AckEveryRecords   int
	AckDelay          time.Duration
}

type recordFlushRequest struct {
	targetSeq uint64
	forceAck  bool
	done      chan error
}

type recordObservability struct {
	batchFramesSent       atomic.Int64
	ackFramesSent         atomic.Int64
	errorFramesSent       atomic.Int64
	batchFramesReceived   atomic.Int64
	ackFramesReceived     atomic.Int64
	errorFramesReceived   atomic.Int64
	piggybackAckSent      atomic.Int64
	piggybackAckReceived  atomic.Int64
	barrierCount          atomic.Int64
	barrierFlushWaitNanos atomic.Int64
	barrierApplyWaitNanos atomic.Int64
}

type recordStream struct {
	stream      Stream
	ctx         context.Context
	cancel      context.CancelFunc
	cfg         recordConfig
	writeMu     sync.Mutex
	sendCh      chan recordOutboundMessage
	flushCh     chan recordFlushRequest
	recvCh      chan RecordMessage
	ackCh       chan struct{}
	readerCh    chan struct{}
	useBatchAck bool
	obs         recordObservability

	mu sync.Mutex

	stateNotify chan struct{}

	runtime           *recordRuntime
	runtimeKey        string
	runtimeWatchOnce  sync.Once
	runtimeDetachOnce sync.Once

	nextOutboundSeq     uint64
	enqueuedOutboundSeq uint64
	flushedOutboundSeq  uint64
	ackedOutboundSeq    uint64
	outstandingRecords  int
	outstandingBytes    int
	outstandingSizes    map[uint64]int
	outboundClosed      bool

	inboundReceivedSeq uint64
	inboundAppliedSeq  uint64
	inboundApplied     map[uint64]struct{}
	inboundAckSentSeq  uint64
	maxPendingApply    int

	remoteClosed bool
	readErr      error
	terminalErr  error
}

var (
	errRecordStreamNil            = errors.New("record stream is nil")
	errRecordRuntimeNil           = errors.New("record runtime is nil")
	errRecordHandlerNotConfigured = errors.New("record handler is not configured")
	errRecordWriteClosed          = errors.New("record stream write side is closed")
	errRecordSeqNotReceived       = errors.New("record sequence not received")
)

func normalizeRecordOpenOptions(opt RecordOpenOptions) RecordOpenOptions {
	if opt.MaxBatchRecords <= 0 {
		opt.MaxBatchRecords = defaultRecordMaxBatchRecords
	}
	if opt.MaxBatchBytes <= 0 {
		opt.MaxBatchBytes = defaultRecordMaxBatchBytes
	}
	if opt.MaxBatchDelay <= 0 {
		opt.MaxBatchDelay = defaultRecordMaxBatchDelay
	}
	if opt.MaxUnackedRecords <= 0 {
		opt.MaxUnackedRecords = defaultRecordMaxUnackedRecords
	}
	if opt.MaxUnackedBytes <= 0 {
		opt.MaxUnackedBytes = defaultRecordMaxUnackedBytes
	}
	if opt.InboundQueueLimit <= 0 {
		opt.InboundQueueLimit = defaultRecordInboundQueueLimit
	}
	if opt.AckEveryRecords <= 0 {
		opt.AckEveryRecords = defaultRecordAckEveryRecords
	}
	if opt.AckDelay <= 0 {
		opt.AckDelay = defaultRecordAckDelay
	}
	opt.Stream = normalizeRecordStreamOpenOptions(opt.Stream)
	return opt
}

func recordConfigFromOptions(opt RecordOpenOptions) recordConfig {
	return recordConfig{
		MaxBatchRecords:   opt.MaxBatchRecords,
		MaxBatchBytes:     opt.MaxBatchBytes,
		MaxBatchDelay:     opt.MaxBatchDelay,
		MaxUnackedRecords: opt.MaxUnackedRecords,
		MaxUnackedBytes:   opt.MaxUnackedBytes,
		InboundQueueLimit: opt.InboundQueueLimit,
		AckEveryRecords:   opt.AckEveryRecords,
		AckDelay:          opt.AckDelay,
	}
}

func normalizeRecordStreamOpenOptions(opt StreamOpenOptions) StreamOpenOptions {
	opt.Channel = StreamRecordChannel
	opt.Metadata = advertiseRecordStreamOpenMetadata(opt.Metadata)
	return opt
}

func WrapStreamAsRecord(stream Stream, opt RecordOpenOptions) (RecordStream, error) {
	if stream == nil {
		return nil, errRecordStreamNil
	}
	opt = normalizeRecordOpenOptions(opt)
	parent := stream.Context()
	if parent == nil {
		parent = context.Background()
	}
	ctx, cancel := context.WithCancel(parent)
	record := &recordStream{
		stream:      stream,
		ctx:         ctx,
		cancel:      cancel,
		cfg:         recordConfigFromOptions(opt),
		sendCh:      make(chan recordOutboundMessage, opt.MaxBatchRecords*2),
		flushCh:     make(chan recordFlushRequest),
		recvCh:      make(chan RecordMessage, opt.InboundQueueLimit),
		ackCh:       make(chan struct{}, 1),
		readerCh:    make(chan struct{}),
		useBatchAck: recordStreamUseBatchAck(stream.Metadata()),

		stateNotify:      make(chan struct{}),
		outstandingSizes: make(map[uint64]int),
		inboundApplied:   make(map[uint64]struct{}),
	}
	go record.writerLoop()
	go record.readLoop()
	return record, nil
}

func (r *recordStream) ID() string {
	if r == nil || r.stream == nil {
		return ""
	}
	return r.stream.ID()
}

func (r *recordStream) Metadata() StreamMetadata {
	if r == nil || r.stream == nil {
		return nil
	}
	return cloneStreamMetadata(r.stream.Metadata())
}

func (r *recordStream) Context() context.Context {
	if r == nil {
		return context.Background()
	}
	return r.ctx
}

func (r *recordStream) WriteRecord(ctx context.Context, payload []byte) (uint64, error) {
	if r == nil {
		return 0, errRecordStreamNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	size := len(payload)
	for {
		r.mu.Lock()
		if err := r.streamErrorLocked(); err != nil {
			r.mu.Unlock()
			return 0, err
		}
		if r.outboundClosed {
			r.mu.Unlock()
			return 0, errRecordWriteClosed
		}
		if r.outstandingRecords >= r.cfg.MaxUnackedRecords || r.outstandingBytes+size > r.cfg.MaxUnackedBytes {
			wait := r.stateNotify
			r.mu.Unlock()
			select {
			case <-r.ctx.Done():
				return 0, r.streamError()
			case <-ctx.Done():
				return 0, ctx.Err()
			case <-wait:
			}
			continue
		}
		r.nextOutboundSeq++
		msg := recordOutboundMessage{
			Seq:     r.nextOutboundSeq,
			Payload: append([]byte(nil), payload...),
		}
		r.outstandingRecords++
		r.outstandingBytes += size
		r.outstandingSizes[msg.Seq] = size
		select {
		case <-r.ctx.Done():
			r.rollbackReservedOutboundLocked(msg.Seq)
			err := r.streamErrorLocked()
			r.mu.Unlock()
			return 0, err
		case <-ctx.Done():
			r.rollbackReservedOutboundLocked(msg.Seq)
			r.mu.Unlock()
			return 0, ctx.Err()
		case r.sendCh <- msg:
			r.enqueuedOutboundSeq = msg.Seq
			r.signalStateLocked()
			r.mu.Unlock()
			return msg.Seq, nil
		}
	}
}

func (r *recordStream) Flush(ctx context.Context) error {
	if r == nil {
		return errRecordStreamNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	if err := r.streamError(); err != nil {
		return err
	}
	req := recordFlushRequest{
		targetSeq: r.flushTargetSeq(),
		done:      make(chan error, 1),
	}
	select {
	case <-r.ctx.Done():
		return r.streamError()
	case <-ctx.Done():
		return ctx.Err()
	case r.flushCh <- req:
	}
	select {
	case <-r.ctx.Done():
		return r.streamError()
	case <-ctx.Done():
		return ctx.Err()
	case err := <-req.done:
		return err
	}
}

func (r *recordStream) Barrier(ctx context.Context) (uint64, error) {
	if r == nil {
		return 0, errRecordStreamNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	return r.BarrierTo(ctx, r.flushTargetSeq())
}

func (r *recordStream) BarrierTo(ctx context.Context, target uint64) (uint64, error) {
	if r == nil {
		return 0, errRecordStreamNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	current := r.flushTargetSeq()
	if target == 0 {
		target = current
	}
	if target > current {
		return 0, errRecordSeqInvalid
	}
	r.obs.barrierCount.Add(1)
	flushStart := time.Now()
	err := r.Flush(ctx)
	r.obs.barrierFlushWaitNanos.Add(time.Since(flushStart).Nanoseconds())
	if err != nil {
		return 0, err
	}
	if target == 0 {
		return 0, nil
	}
	applyStart := time.Now()
	err = r.waitAckedAtLeast(ctx, target)
	r.obs.barrierApplyWaitNanos.Add(time.Since(applyStart).Nanoseconds())
	if err != nil {
		return 0, err
	}
	return target, nil
}

func (r *recordStream) ReadRecord(ctx context.Context) (RecordMessage, error) {
	if r == nil {
		return RecordMessage{}, errRecordStreamNil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	select {
	case <-r.ctx.Done():
		return RecordMessage{}, r.readError()
	case <-ctx.Done():
		return RecordMessage{}, ctx.Err()
	case msg, ok := <-r.recvCh:
		if ok {
			return msg, nil
		}
		return RecordMessage{}, r.readError()
	}
}

func (r *recordStream) AckRecord(seq uint64) error {
	if r == nil {
		return errRecordStreamNil
	}
	if seq == 0 {
		return errRecordSeqInvalid
	}
	r.mu.Lock()
	if seq > r.inboundReceivedSeq {
		r.mu.Unlock()
		return errRecordSeqNotReceived
	}
	if seq <= r.inboundAppliedSeq {
		r.mu.Unlock()
		return nil
	}
	r.inboundApplied[seq] = struct{}{}
	advanced := false
	for {
		next := r.inboundAppliedSeq + 1
		if _, ok := r.inboundApplied[next]; !ok {
			break
		}
		delete(r.inboundApplied, next)
		r.inboundAppliedSeq = next
		advanced = true
	}
	if advanced {
		r.signalStateLocked()
	}
	r.mu.Unlock()
	if advanced {
		r.notifyAckLoop()
	}
	return nil
}

func (r *recordStream) FailRecord(seq uint64, failure RecordFailure) error {
	if r == nil {
		return errRecordStreamNil
	}
	if seq == 0 {
		return errRecordSeqInvalid
	}
	if failure.FailedSeq == 0 {
		failure.FailedSeq = seq
	}
	if failure.Code == "" {
		failure.Code = RecordErrorCodeApplyFailed
	}
	err := r.sendFailureFrame(failure)
	if err != nil {
		return err
	}
	r.setTerminalError(failure)
	return r.stream.Reset(failure)
}

func (r *recordStream) CloseWrite() error {
	if r == nil {
		return errRecordStreamNil
	}
	if err := r.Flush(context.Background()); err != nil {
		return err
	}
	if err := r.flushAckNow(); err != nil {
		return err
	}
	r.mu.Lock()
	r.outboundClosed = true
	r.signalStateLocked()
	r.mu.Unlock()
	return r.stream.CloseWrite()
}

func (r *recordStream) Close() error {
	if r == nil {
		return nil
	}
	_ = r.flushAckNow()
	r.cancel()
	return r.stream.Close()
}

func (r *recordStream) Reset(err error) error {
	if r == nil {
		return nil
	}
	r.setTerminalError(err)
	return r.stream.Reset(err)
}

func (r *recordStream) flushTargetSeq() uint64 {
	if r == nil {
		return 0
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.enqueuedOutboundSeq
}

func (r *recordStream) waitAckedAtLeast(ctx context.Context, target uint64) error {
	for {
		r.mu.Lock()
		if err := r.streamErrorLocked(); err != nil {
			r.mu.Unlock()
			return err
		}
		if r.ackedOutboundSeq >= target {
			r.mu.Unlock()
			return nil
		}
		if r.remoteClosed {
			r.mu.Unlock()
			return io.EOF
		}
		wait := r.stateNotify
		r.mu.Unlock()
		select {
		case <-r.ctx.Done():
			return r.streamError()
		case <-ctx.Done():
			return ctx.Err()
		case <-wait:
		}
	}
}

func (r *recordStream) writerLoop() {
	var (
		batch        []recordOutboundMessage
		batches      int
		bytes        int
		batchTimer   *time.Timer
		batchTimerCh <-chan time.Time
		ackTimer     *time.Timer
		ackTimerCh   <-chan time.Time
	)
	stopBatchTimer := func() {
		if batchTimer == nil {
			return
		}
		if !batchTimer.Stop() {
			select {
			case <-batchTimer.C:
			default:
			}
		}
		batchTimerCh = nil
	}
	stopAckTimer := func() {
		if ackTimer == nil {
			return
		}
		if !ackTimer.Stop() {
			select {
			case <-ackTimer.C:
			default:
			}
		}
		ackTimerCh = nil
	}
	scheduleAck := func(hasPendingBatch bool, force bool) (uint64, bool) {
		ackSeq := r.pendingAckSeq()
		if ackSeq == 0 {
			stopAckTimer()
			return 0, false
		}
		if force {
			stopAckTimer()
			return ackSeq, true
		}
		if hasPendingBatch && r.useBatchAck {
			stopAckTimer()
			return 0, false
		}
		if r.shouldSendAckNow() || r.cfg.AckDelay <= 0 {
			stopAckTimer()
			return ackSeq, true
		}
		if ackTimer == nil {
			ackTimer = time.NewTimer(r.cfg.AckDelay)
		} else {
			ackTimer.Reset(r.cfg.AckDelay)
		}
		ackTimerCh = ackTimer.C
		return 0, false
	}
	sendStandaloneAck := func(ackSeq uint64) error {
		if ackSeq == 0 {
			return nil
		}
		payload, err := encodeRecordAckFrame(ackSeq)
		if err != nil {
			return err
		}
		if err := r.writePayloadFrame(payload); err != nil {
			return err
		}
		r.obs.ackFramesSent.Add(1)
		r.markAckSent(ackSeq)
		return nil
	}
	flushBatch := func() error {
		if len(batch) == 0 {
			return nil
		}
		ackSeq := r.pendingAckSeq()
		payload, err := encodeRecordBatchFrame(batch, ackSeq, r.useBatchAck)
		if err != nil {
			return err
		}
		if err := r.writePayloadFrame(payload); err != nil {
			return err
		}
		r.obs.batchFramesSent.Add(1)
		if r.useBatchAck && ackSeq != 0 {
			r.obs.piggybackAckSent.Add(1)
			r.markAckSent(ackSeq)
		}
		r.markFlushed(batch[len(batch)-1].Seq)
		batch = nil
		batches = 0
		bytes = 0
		stopBatchTimer()
		if ackSeq, sendNow := scheduleAck(false, false); sendNow {
			return sendStandaloneAck(ackSeq)
		}
		return nil
	}
	flushUntil := func(target uint64) error {
		for {
			if target == 0 {
				return flushBatch()
			}
			if r.flushedAtLeast(target) {
				return nil
			}
			if len(batch) > 0 && batch[len(batch)-1].Seq >= target {
				if err := flushBatch(); err != nil {
					return err
				}
				if r.flushedAtLeast(target) {
					return nil
				}
				continue
			}
			req, ok := r.nextOutboundForFlush()
			if !ok {
				return r.streamError()
			}
			batch = append(batch, req)
			batches++
			bytes += len(req.Payload)
			if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
				if err := flushBatch(); err != nil {
					return err
				}
			}
		}
	}
	for {
		select {
		case <-r.ctx.Done():
			return
		case req := <-r.sendCh:
			batch = append(batch, req)
			batches++
			bytes += len(req.Payload)
			if len(batch) == 1 && r.cfg.MaxBatchDelay > 0 {
				if batchTimer == nil {
					batchTimer = time.NewTimer(r.cfg.MaxBatchDelay)
				} else {
					batchTimer.Reset(r.cfg.MaxBatchDelay)
				}
				batchTimerCh = batchTimer.C
			}
			if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
				if err := flushBatch(); err != nil {
					r.setTerminalError(err)
					return
				}
				continue
			}
			if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
				if err := sendStandaloneAck(ackSeq); err != nil {
					r.setTerminalError(err)
					return
				}
			}
		case req := <-r.flushCh:
			err := flushUntil(req.targetSeq)
			if err == nil && req.forceAck {
				if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
					err = sendStandaloneAck(ackSeq)
				}
			}
			req.done <- err
		case <-batchTimerCh:
			if err := flushBatch(); err != nil {
				r.setTerminalError(err)
				return
			}
		case <-r.ackCh:
			if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
				if err := sendStandaloneAck(ackSeq); err != nil {
					r.setTerminalError(err)
					return
				}
			}
		case <-ackTimerCh:
			stopAckTimer()
			if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
				if err := sendStandaloneAck(ackSeq); err != nil {
					r.setTerminalError(err)
					return
				}
			}
		}
	}
}

func (r *recordStream) readLoop() {
	defer close(r.recvCh)
	defer close(r.readerCh)
	for {
		payload, err := readTransferFrame(r.stream)
		if err != nil {
			if errors.Is(err, io.EOF) {
				r.markRemoteClosed(nil)
				return
			}
			r.setReadError(err)
			return
		}
		frame, err := decodeRecordFrame(payload)
		if err != nil {
			_ = r.sendFailureFrame(RecordFailure{
				FailedSeq: r.nextInboundFailureSeq(),
				Code:      RecordErrorCodeProtocol,
				Message:   err.Error(),
			})
			r.setReadError(err)
			_ = r.stream.Reset(err)
			return
		}
		switch frame.Type {
		case recordFrameTypeBatch:
			r.obs.batchFramesReceived.Add(1)
			if frame.AckSeq != 0 {
				r.obs.piggybackAckReceived.Add(1)
				if err := r.handleAckFrame(frame.AckSeq); err != nil {
					r.setReadError(err)
					_ = r.stream.Reset(err)
					return
				}
			}
			if err := r.handleBatchFrame(frame.Batch); err != nil {
				_ = r.sendFailureFrame(RecordFailure{
					FailedSeq: r.nextInboundFailureSeq(),
					Code:      RecordErrorCodeProtocol,
					Message:   err.Error(),
				})
				r.setReadError(err)
				_ = r.stream.Reset(err)
				return
			}
		case recordFrameTypeAck:
			r.obs.ackFramesReceived.Add(1)
			if err := r.handleAckFrame(frame.AckSeq); err != nil {
				r.setReadError(err)
				_ = r.stream.Reset(err)
				return
			}
		case recordFrameTypeError:
			r.obs.errorFramesReceived.Add(1)
			r.setReadError(frame.Failure)
			return
		default:
			r.setReadError(errRecordFrameInvalid)
			return
		}
	}
}

func (r *recordStream) handleBatchFrame(batch []recordOutboundMessage) error {
	if len(batch) == 0 {
		return errRecordFrameInvalid
	}
	r.mu.Lock()
	expected := r.inboundReceivedSeq + 1
	if batch[0].Seq != expected {
		r.mu.Unlock()
		return errRecordSeqInvalid
	}
	lastSeq := batch[len(batch)-1].Seq
	r.inboundReceivedSeq = lastSeq
	r.updatePendingApplyLocked()
	r.signalStateLocked()
	r.mu.Unlock()
	for _, item := range batch {
		select {
		case <-r.ctx.Done():
			return r.streamError()
		case r.recvCh <- RecordMessage{Seq: item.Seq, Payload: item.Payload}:
		}
	}
	return nil
}

func (r *recordStream) handleAckFrame(ackSeq uint64) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	if ackSeq < r.ackedOutboundSeq || ackSeq > r.nextOutboundSeq {
		return errRecordSeqInvalid
	}
	for seq := r.ackedOutboundSeq + 1; seq <= ackSeq; seq++ {
		if size, ok := r.outstandingSizes[seq]; ok {
			delete(r.outstandingSizes, seq)
			r.outstandingBytes -= size
			if r.outstandingBytes < 0 {
				r.outstandingBytes = 0
			}
			r.outstandingRecords--
			if r.outstandingRecords < 0 {
				r.outstandingRecords = 0
			}
		}
	}
	r.ackedOutboundSeq = ackSeq
	r.signalStateLocked()
	return nil
}

func (r *recordStream) markFlushed(seq uint64) {
	if r == nil || seq == 0 {
		return
	}
	r.mu.Lock()
	if seq > r.flushedOutboundSeq {
		r.flushedOutboundSeq = seq
		r.signalStateLocked()
	}
	r.mu.Unlock()
}

func (r *recordStream) flushedAtLeast(target uint64) bool {
	if r == nil || target == 0 {
		return true
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.flushedOutboundSeq >= target
}

func (r *recordStream) markRemoteClosed(err error) {
	r.mu.Lock()
	r.remoteClosed = true
	if err != nil && r.readErr == nil {
		r.readErr = err
	}
	r.signalStateLocked()
	r.mu.Unlock()
}

func (r *recordStream) setReadError(err error) {
	if err == nil {
		return
	}
	r.mu.Lock()
	if r.readErr == nil {
		r.readErr = err
	}
	r.signalStateLocked()
	r.mu.Unlock()
	r.cancel()
}

func (r *recordStream) setTerminalError(err error) {
	if err == nil {
		return
	}
	r.mu.Lock()
	if r.terminalErr == nil {
		r.terminalErr = err
	}
	if r.readErr == nil {
		r.readErr = err
	}
	r.signalStateLocked()
	r.mu.Unlock()
	r.cancel()
}

func (r *recordStream) rollbackReservedOutboundLocked(seq uint64) {
	if r == nil || seq == 0 {
		return
	}
	if size, ok := r.outstandingSizes[seq]; ok {
		delete(r.outstandingSizes, seq)
		r.outstandingBytes -= size
		if r.outstandingBytes < 0 {
			r.outstandingBytes = 0
		}
		r.outstandingRecords--
		if r.outstandingRecords < 0 {
			r.outstandingRecords = 0
		}
	}
	if r.nextOutboundSeq == seq {
		r.nextOutboundSeq--
	}
	r.signalStateLocked()
}

func (r *recordStream) readError() error {
	if r == nil {
		return errRecordStreamNil
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.readErr != nil {
		return r.readErr
	}
	if r.terminalErr != nil {
		return r.terminalErr
	}
	return io.EOF
}

func (r *recordStream) streamError() error {
	if r == nil {
		return errRecordStreamNil
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.streamErrorLocked()
}

func (r *recordStream) streamErrorLocked() error {
	if r.readErr != nil {
		return r.readErr
	}
	if r.terminalErr != nil {
		return r.terminalErr
	}
	return nil
}

func (r *recordStream) shouldSendAckNow() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.inboundAppliedSeq > r.inboundAckSentSeq && int(r.inboundAppliedSeq-r.inboundAckSentSeq) >= r.cfg.AckEveryRecords
}

func (r *recordStream) pendingAckSeq() uint64 {
	if r == nil {
		return 0
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.inboundAppliedSeq <= r.inboundAckSentSeq {
		return 0
	}
	return r.inboundAppliedSeq
}

func (r *recordStream) markAckSent(ackSeq uint64) {
	if r == nil || ackSeq == 0 {
		return
	}
	r.mu.Lock()
	if ackSeq > r.inboundAckSentSeq {
		r.inboundAckSentSeq = ackSeq
		r.signalStateLocked()
	}
	r.mu.Unlock()
}

func (r *recordStream) flushAckNow() error {
	if r == nil {
		return errRecordStreamNil
	}
	req := recordFlushRequest{
		forceAck: true,
		done:     make(chan error, 1),
	}
	select {
	case <-r.ctx.Done():
		return r.streamError()
	case r.flushCh <- req:
	}
	select {
	case <-r.ctx.Done():
		return r.streamError()
	case err := <-req.done:
		return err
	}
}

func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
	payload, err := encodeRecordErrorFrame(failure)
	if err != nil {
		return err
	}
	if err := r.writePayloadFrame(payload); err != nil {
		return err
	}
	r.obs.errorFramesSent.Add(1)
	return nil
}

func (r *recordStream) writePayloadFrame(payload []byte) error {
	if r == nil {
		return errRecordStreamNil
	}
	if payload == nil {
		return nil
	}
	frame := buildTransferFrame(payload)
	r.writeMu.Lock()
	defer r.writeMu.Unlock()
	return writeTransferFrames(r.stream, frame)
}

func (r *recordStream) notifyAckLoop() {
	if r == nil {
		return
	}
	select {
	case r.ackCh <- struct{}{}:
	default:
	}
}

func (r *recordStream) nextOutboundForFlush() (recordOutboundMessage, bool) {
	if r == nil {
		return recordOutboundMessage{}, false
	}
	select {
	case <-r.ctx.Done():
		return recordOutboundMessage{}, false
	case req := <-r.sendCh:
		return req, true
	}
}

func (r *recordStream) nextInboundFailureSeq() uint64 {
	if r == nil {
		return 1
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.inboundReceivedSeq + 1
}

func (r *recordStream) signalStateLocked() {
	close(r.stateNotify)
	r.stateNotify = make(chan struct{})
}

func (r *recordStream) updatePendingApplyLocked() {
	if r == nil {
		return
	}
	pending := recordPendingCount(r.inboundReceivedSeq, r.inboundAppliedSeq)
	if pending > r.maxPendingApply {
		r.maxPendingApply = pending
	}
}