feat: 完善 RecordStream 的协议协商、运行观测与文档说明

- 将 RecordStream 出站路径收敛为单 writer loop
  - 支持在 batch header 中 piggyback AckSeq,保留独立 ack 作为兼容回退
  - 增加 record stream 打开阶段能力协商,支持 mixed-version peer 自动降级
  - 补充 RecordSnapshot 与 diagnostics summary 的 record-plane 观测项
  - 增加 batch/ack/error frame、piggyback ack、barrier 等待拆分与 apply backlog 指标
  - 收紧 TransportConn detach 后的 runtime snapshot 语义
  - 补充 README 中的 RecordStream 语义、兼容行为与诊断快照说明
  - 补充相关单测与 race 回归验证
This commit is contained in:
兔子 2026-04-15 19:52:45 +08:00
parent 09d972c7b7
commit 7ed3dd5b37
Signed by: b612
GPG Key ID: 99DD2222B612B612
16 changed files with 1341 additions and 145 deletions

View File

@ -9,6 +9,7 @@
- 记录流数据面:`OpenRecordStream` - 记录流数据面:`OpenRecordStream`
- 批量数据面:`OpenBulk``shared` / `dedicated` - 批量数据面:`OpenBulk``shared` / `dedicated`
- 文件传输内核transfer control / progress / resume - 文件传输内核transfer control / progress / resume
- 观测面runtime snapshot / diagnostics summary
- 会话模型:`LogicalConn`(逻辑会话)与 `TransportConn`(物理承载)分离 - 会话模型:`LogicalConn`(逻辑会话)与 `TransportConn`(物理承载)分离
## 版本要求 ## 版本要求
@ -82,6 +83,51 @@ func main() {
} }
``` ```
## RecordStream 说明
`RecordStream` 构建在 `Stream` 之上,适合“有边界的顺序记录”场景。
- 写入入口:`OpenRecordStream``WriteRecord`
- 接收入口:`ReadRecord`
- 确认入口:`AckRecord`
- 检查点:`Barrier``BarrierTo`
- 错误回包:`RecordFailure`
确认语义:
- `AckRecord` 表示“该序号及其之前的连续记录已完成 apply”不是“已收到”
- `Barrier` / `BarrierTo` 等待的是对端 `apply-complete` 的最大连续序号
- `RecordFailure` 会返回 `FailedSeq``Code``Retryable``Message`
兼容与传输:
- record stream 在打开阶段协商 batch ack 能力
- 双端都支持时,累计 `AckSeq` 会随 batch header piggyback 发送
- 对端不支持时,自动回退到独立 ack frame
- mixed-version peer 可以互通,不要求双方同时升级
## 诊断快照
顶层诊断入口:
- `GetClientDiagnosticsSnapshot`
- `GetServerDiagnosticsSnapshot`
快照内容:
- 会话运行态client / server runtime
- 数据面快照:`StreamSnapshot``BulkSnapshot``RecordSnapshot`
- 文件传输快照:`TransferSnapshot`
- 汇总视图:`DiagnosticsSummary`
`RecordSnapshot` / `DiagnosticsSummary.RecordTelemetry` 当前覆盖:
- batch / ack / error frame 收发计数
- piggyback ack 命中计数
- barrier 等待时间拆分:`flush` / `apply`
- `outstanding records/bytes`
- `pending apply / pending ack / peak pending apply`
## 传输与 IPC ## 传输与 IPC
- `tcp` - `tcp`

View File

@ -24,6 +24,7 @@ func (c *ClientCommon) OpenRecordStream(ctx context.Context, opt RecordOpenOptio
_ = stream.Reset(err) _ = stream.Reset(err)
return nil, err return nil, err
} }
bindRecordRuntime(record, c.getRecordRuntime())
return record, nil return record, nil
} }
@ -51,6 +52,7 @@ func (c *ClientCommon) claimInboundRecordStream(stream *streamHandle) (bool, err
if err != nil { if err != nil {
return true, err return true, err
} }
bindRecordRuntime(record, runtime)
info := RecordAcceptInfo{ info := RecordAcceptInfo{
ID: stream.ID(), ID: stream.ID(),
Metadata: stream.Metadata(), Metadata: stream.Metadata(),

View File

@ -35,6 +35,7 @@ func (c *ClientCommon) OpenStream(ctx context.Context, opt StreamOpenOptions) (S
if resp.DataID != 0 { if resp.DataID != 0 {
req.DataID = resp.DataID req.DataID = resp.DataID
} }
req.Metadata = mergeStreamMetadata(req.Metadata, resp.Metadata)
stream := newStreamHandle(c.clientStopContextSnapshot(), runtime, clientFileScope(), req, c.currentClientSessionEpoch(), nil, nil, resp.TransportGeneration, clientStreamCloseSender(c), clientStreamResetSender(c), clientStreamDataSender(c, c.currentClientSessionEpoch()), runtime.configSnapshot()) stream := newStreamHandle(c.clientStopContextSnapshot(), runtime, clientFileScope(), req, c.currentClientSessionEpoch(), nil, nil, resp.TransportGeneration, clientStreamCloseSender(c), clientStreamResetSender(c), clientStreamDataSender(c, c.currentClientSessionEpoch()), runtime.configSnapshot())
stream.setClientSnapshotOwner(c) stream.setClientSnapshotOwner(c)
stream.setAddrSnapshot(c.clientStreamAddrSnapshot()) stream.setAddrSnapshot(c.clientStreamAddrSnapshot())

View File

@ -34,6 +34,27 @@ type DiagnosticsTransferTelemetrySummary struct {
CommitWaitRatio float64 CommitWaitRatio float64
} }
type DiagnosticsRecordTelemetrySummary struct {
BatchFramesSent int64
AckFramesSent int64
ErrorFramesSent int64
BatchFramesReceived int64
AckFramesReceived int64
ErrorFramesReceived int64
FrameSendCount int64
FrameReceiveCount int64
PiggybackAckSent int64
PiggybackAckReceived int64
BarrierCount int64
BarrierFlushWaitDuration time.Duration
BarrierApplyWaitDuration time.Duration
OutstandingRecords int
OutstandingBytes int
PendingApplyRecords int
PendingAckRecords int
PeakPendingApplyRecords int
}
type DiagnosticsSummary struct { type DiagnosticsSummary struct {
LogicalCount int LogicalCount int
CurrentTransportCount int CurrentTransportCount int
@ -49,6 +70,11 @@ type DiagnosticsSummary struct {
StaleBulkCount int StaleBulkCount int
ResetBulkCount int ResetBulkCount int
RecordCount int
ActiveRecordCount int
StaleRecordCount int
ResetRecordCount int
TransferCount int TransferCount int
ActiveTransferCount int ActiveTransferCount int
PausedTransferCount int PausedTransferCount int
@ -58,6 +84,8 @@ type DiagnosticsSummary struct {
StreamResetCauses DiagnosticsResetCauseSummary StreamResetCauses DiagnosticsResetCauseSummary
BulkResetCauses DiagnosticsResetCauseSummary BulkResetCauses DiagnosticsResetCauseSummary
RecordResetCauses DiagnosticsResetCauseSummary
RecordTelemetry DiagnosticsRecordTelemetrySummary
TransferTelemetry DiagnosticsTransferTelemetrySummary TransferTelemetry DiagnosticsTransferTelemetrySummary
} }
@ -65,6 +93,7 @@ type ClientDiagnosticsSnapshot struct {
Runtime ClientRuntimeSnapshot Runtime ClientRuntimeSnapshot
Streams []StreamSnapshot Streams []StreamSnapshot
Bulks []BulkSnapshot Bulks []BulkSnapshot
Records []RecordSnapshot
Transfers []TransferSnapshot Transfers []TransferSnapshot
Summary DiagnosticsSummary Summary DiagnosticsSummary
} }
@ -75,6 +104,7 @@ type ServerDiagnosticsSnapshot struct {
CurrentTransports []TransportConnRuntimeSnapshot CurrentTransports []TransportConnRuntimeSnapshot
Streams []StreamSnapshot Streams []StreamSnapshot
Bulks []BulkSnapshot Bulks []BulkSnapshot
Records []RecordSnapshot
Transfers []TransferSnapshot Transfers []TransferSnapshot
Summary DiagnosticsSummary Summary DiagnosticsSummary
} }
@ -100,6 +130,10 @@ func GetClientDiagnosticsSnapshot(c Client) (ClientDiagnosticsSnapshot, error) {
if err != nil { if err != nil {
return ClientDiagnosticsSnapshot{}, err return ClientDiagnosticsSnapshot{}, err
} }
records, err := GetClientRecordSnapshots(c)
if err != nil {
return ClientDiagnosticsSnapshot{}, err
}
transfers, err := GetClientTransferSnapshots(c) transfers, err := GetClientTransferSnapshots(c)
if err != nil { if err != nil {
return ClientDiagnosticsSnapshot{}, err return ClientDiagnosticsSnapshot{}, err
@ -108,6 +142,7 @@ func GetClientDiagnosticsSnapshot(c Client) (ClientDiagnosticsSnapshot, error) {
Runtime: runtime, Runtime: runtime,
Streams: streams, Streams: streams,
Bulks: bulks, Bulks: bulks,
Records: records,
Transfers: transfers, Transfers: transfers,
} }
snapshot.Summary = summarizeClientDiagnosticsSnapshot(snapshot) snapshot.Summary = summarizeClientDiagnosticsSnapshot(snapshot)
@ -138,6 +173,10 @@ func GetServerDiagnosticsSnapshot(s Server) (ServerDiagnosticsSnapshot, error) {
if err != nil { if err != nil {
return ServerDiagnosticsSnapshot{}, err return ServerDiagnosticsSnapshot{}, err
} }
records, err := GetServerRecordSnapshots(s)
if err != nil {
return ServerDiagnosticsSnapshot{}, err
}
transfers, err := GetServerTransferSnapshots(s) transfers, err := GetServerTransferSnapshots(s)
if err != nil { if err != nil {
return ServerDiagnosticsSnapshot{}, err return ServerDiagnosticsSnapshot{}, err
@ -148,6 +187,7 @@ func GetServerDiagnosticsSnapshot(s Server) (ServerDiagnosticsSnapshot, error) {
CurrentTransports: transports, CurrentTransports: transports,
Streams: streams, Streams: streams,
Bulks: bulks, Bulks: bulks,
Records: records,
Transfers: transfers, Transfers: transfers,
} }
snapshot.Summary = summarizeServerDiagnosticsSnapshot(snapshot) snapshot.Summary = summarizeServerDiagnosticsSnapshot(snapshot)
@ -203,6 +243,7 @@ func summarizeClientDiagnosticsSnapshot(snapshot ClientDiagnosticsSnapshot) Diag
} }
summarizeStreamSnapshots(&summary, snapshot.Streams) summarizeStreamSnapshots(&summary, snapshot.Streams)
summarizeBulkSnapshots(&summary, snapshot.Bulks) summarizeBulkSnapshots(&summary, snapshot.Bulks)
summarizeRecordSnapshots(&summary, snapshot.Records)
summarizeTransferSnapshots(&summary, snapshot.Transfers) summarizeTransferSnapshots(&summary, snapshot.Transfers)
return summary return summary
} }
@ -214,6 +255,7 @@ func summarizeServerDiagnosticsSnapshot(snapshot ServerDiagnosticsSnapshot) Diag
} }
summarizeStreamSnapshots(&summary, snapshot.Streams) summarizeStreamSnapshots(&summary, snapshot.Streams)
summarizeBulkSnapshots(&summary, snapshot.Bulks) summarizeBulkSnapshots(&summary, snapshot.Bulks)
summarizeRecordSnapshots(&summary, snapshot.Records)
summarizeTransferSnapshots(&summary, snapshot.Transfers) summarizeTransferSnapshots(&summary, snapshot.Transfers)
return summary return summary
} }
@ -266,6 +308,27 @@ func summarizeBulkSnapshots(summary *DiagnosticsSummary, snapshots []BulkSnapsho
} }
} }
func summarizeRecordSnapshots(summary *DiagnosticsSummary, snapshots []RecordSnapshot) {
if summary == nil {
return
}
summary.RecordCount = len(snapshots)
for _, snapshot := range snapshots {
switch {
case snapshot.ResetError != "":
summary.ResetRecordCount++
accumulateDiagnosticsResetCause(&summary.RecordResetCauses, snapshot.ResetError, "")
case recordSnapshotFinished(snapshot):
case recordSnapshotBoundActive(snapshot):
summary.ActiveRecordCount++
default:
summary.StaleRecordCount++
}
accumulateDiagnosticsRecordTelemetry(&summary.RecordTelemetry, snapshot)
}
finalizeDiagnosticsRecordTelemetry(&summary.RecordTelemetry)
}
func summarizeTransferSnapshots(summary *DiagnosticsSummary, snapshots []TransferSnapshot) { func summarizeTransferSnapshots(summary *DiagnosticsSummary, snapshots []TransferSnapshot) {
if summary == nil { if summary == nil {
return return
@ -297,6 +360,10 @@ func bulkSnapshotFinished(snapshot BulkSnapshot) bool {
return snapshot.ResetError == "" && snapshot.LocalClosed && snapshot.RemoteClosed return snapshot.ResetError == "" && snapshot.LocalClosed && snapshot.RemoteClosed
} }
func recordSnapshotFinished(snapshot RecordSnapshot) bool {
return snapshot.ResetError == "" && snapshot.LocalClosed && snapshot.RemoteClosed
}
func streamSnapshotBoundActive(snapshot StreamSnapshot) bool { func streamSnapshotBoundActive(snapshot StreamSnapshot) bool {
return snapshot.BindingCurrent && snapshot.TransportAttached && snapshot.TransportCurrent return snapshot.BindingCurrent && snapshot.TransportAttached && snapshot.TransportCurrent
} }
@ -305,6 +372,10 @@ func bulkSnapshotBoundActive(snapshot BulkSnapshot) bool {
return snapshot.BindingCurrent && snapshot.TransportAttached && snapshot.TransportCurrent return snapshot.BindingCurrent && snapshot.TransportAttached && snapshot.TransportCurrent
} }
func recordSnapshotBoundActive(snapshot RecordSnapshot) bool {
return snapshot.BindingCurrent && snapshot.TransportAttached && snapshot.TransportCurrent
}
func accumulateDiagnosticsResetCause(summary *DiagnosticsResetCauseSummary, resetError string, backpressureError string) { func accumulateDiagnosticsResetCause(summary *DiagnosticsResetCauseSummary, resetError string, backpressureError string) {
if summary == nil || resetError == "" { if summary == nil || resetError == "" {
return return
@ -362,6 +433,38 @@ func finalizeDiagnosticsTransferTelemetry(summary *DiagnosticsTransferTelemetryS
summary.CommitWaitRatio = durationRatio(summary.CommitWaitDuration, summary.ObservedDuration) summary.CommitWaitRatio = durationRatio(summary.CommitWaitDuration, summary.ObservedDuration)
} }
func accumulateDiagnosticsRecordTelemetry(summary *DiagnosticsRecordTelemetrySummary, snapshot RecordSnapshot) {
if summary == nil {
return
}
summary.BatchFramesSent += snapshot.BatchFramesSent
summary.AckFramesSent += snapshot.AckFramesSent
summary.ErrorFramesSent += snapshot.ErrorFramesSent
summary.BatchFramesReceived += snapshot.BatchFramesReceived
summary.AckFramesReceived += snapshot.AckFramesReceived
summary.ErrorFramesReceived += snapshot.ErrorFramesReceived
summary.PiggybackAckSent += snapshot.PiggybackAckSent
summary.PiggybackAckReceived += snapshot.PiggybackAckReceived
summary.BarrierCount += snapshot.BarrierCount
summary.BarrierFlushWaitDuration += snapshot.BarrierFlushWaitDuration
summary.BarrierApplyWaitDuration += snapshot.BarrierApplyWaitDuration
summary.OutstandingRecords += snapshot.OutstandingRecords
summary.OutstandingBytes += snapshot.OutstandingBytes
summary.PendingApplyRecords += snapshot.PendingApplyRecords
summary.PendingAckRecords += snapshot.PendingAckRecords
if snapshot.PeakPendingApplyRecords > summary.PeakPendingApplyRecords {
summary.PeakPendingApplyRecords = snapshot.PeakPendingApplyRecords
}
}
func finalizeDiagnosticsRecordTelemetry(summary *DiagnosticsRecordTelemetrySummary) {
if summary == nil {
return
}
summary.FrameSendCount = summary.BatchFramesSent + summary.AckFramesSent + summary.ErrorFramesSent
summary.FrameReceiveCount = summary.BatchFramesReceived + summary.AckFramesReceived + summary.ErrorFramesReceived
}
func sortClientConnRuntimeSnapshots(src []ClientConnRuntimeSnapshot) { func sortClientConnRuntimeSnapshots(src []ClientConnRuntimeSnapshot) {
sort.Slice(src, func(i, j int) bool { sort.Slice(src, func(i, j int) bool {
if src[i].ClientID != src[j].ClientID { if src[i].ClientID != src[j].ClientID {

View File

@ -20,7 +20,7 @@ func TestGetClientDiagnosticsSnapshotDefaults(t *testing.T) {
if got, want := snapshot.Runtime.OwnerState, "idle"; got != want { if got, want := snapshot.Runtime.OwnerState, "idle"; got != want {
t.Fatalf("Runtime.OwnerState = %q, want %q", got, want) t.Fatalf("Runtime.OwnerState = %q, want %q", got, want)
} }
if len(snapshot.Streams) != 0 || len(snapshot.Bulks) != 0 || len(snapshot.Transfers) != 0 { if len(snapshot.Streams) != 0 || len(snapshot.Bulks) != 0 || len(snapshot.Records) != 0 || len(snapshot.Transfers) != 0 {
t.Fatalf("default diagnostics should be empty: %+v", snapshot) t.Fatalf("default diagnostics should be empty: %+v", snapshot)
} }
if snapshot.Summary != (DiagnosticsSummary{}) { if snapshot.Summary != (DiagnosticsSummary{}) {
@ -130,6 +130,137 @@ func TestGetClientDiagnosticsSnapshotAggregatesActiveState(t *testing.T) {
_ = bulk.Close() _ = bulk.Close()
} }
func TestGetDiagnosticsSnapshotAggregatesActiveRecordState(t *testing.T) {
server := NewServer().(*ServerCommon)
if err := UseModernPSKServer(server, integrationSharedSecret, integrationModernPSKOptions()); err != nil {
t.Fatalf("UseModernPSKServer failed: %v", err)
}
recordAcceptCh := make(chan RecordAcceptInfo, 1)
recordReleaseCh := make(chan struct{})
recordHandlerDone := make(chan error, 1)
server.SetRecordStreamHandler(func(info RecordAcceptInfo) error {
recordAcceptCh <- info
msg, err := info.RecordStream.ReadRecord(context.Background())
if err != nil {
recordHandlerDone <- err
return err
}
if string(msg.Payload) != "diag-record" {
err = errors.New("unexpected record payload")
recordHandlerDone <- err
return err
}
if err := info.RecordStream.AckRecord(msg.Seq); err != nil {
recordHandlerDone <- err
return err
}
<-recordReleaseCh
err = info.RecordStream.Close()
recordHandlerDone <- err
return err
})
if err := server.Listen("tcp", "127.0.0.1:0"); err != nil {
t.Fatalf("server Listen failed: %v", err)
}
defer func() {
_ = server.Stop()
}()
client := NewClient().(*ClientCommon)
if err := UseModernPSKClient(client, integrationSharedSecret, integrationModernPSKOptions()); err != nil {
t.Fatalf("UseModernPSKClient failed: %v", err)
}
if err := client.Connect("tcp", server.listener.Addr().String()); err != nil {
t.Fatalf("client Connect failed: %v", err)
}
defer func() {
_ = client.Stop()
}()
record, err := client.OpenRecordStream(context.Background(), RecordOpenOptions{
Stream: StreamOpenOptions{ID: "diag-client-record"},
})
if err != nil {
t.Fatalf("client OpenRecordStream failed: %v", err)
}
select {
case <-recordAcceptCh:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting accepted record stream")
}
if _, err := record.WriteRecord(context.Background(), []byte("diag-record")); err != nil {
t.Fatalf("WriteRecord failed: %v", err)
}
if _, err := record.Barrier(context.Background()); err != nil {
t.Fatalf("Barrier failed: %v", err)
}
clientSnapshot, err := GetClientDiagnosticsSnapshot(client)
if err != nil {
t.Fatalf("GetClientDiagnosticsSnapshot failed: %v", err)
}
if got, want := len(clientSnapshot.Records), 1; got != want {
t.Fatalf("client record snapshot count = %d, want %d", got, want)
}
if got, want := clientSnapshot.Summary.RecordCount, 1; got != want {
t.Fatalf("client RecordCount = %d, want %d", got, want)
}
if got, want := clientSnapshot.Summary.ActiveRecordCount, 1; got != want {
t.Fatalf("client ActiveRecordCount = %d, want %d", got, want)
}
clientRecord := clientSnapshot.Records[0]
if got := clientRecord.BatchFramesSent; got < 1 {
t.Fatalf("client BatchFramesSent = %d, want >= 1", got)
}
if got := clientRecord.AckFramesReceived; got < 1 {
t.Fatalf("client AckFramesReceived = %d, want >= 1", got)
}
if got := clientRecord.BarrierCount; got < 1 {
t.Fatalf("client BarrierCount = %d, want >= 1", got)
}
if got := clientSnapshot.Summary.RecordTelemetry.FrameSendCount; got < 1 {
t.Fatalf("client RecordTelemetry.FrameSendCount = %d, want >= 1", got)
}
serverSnapshot, err := GetServerDiagnosticsSnapshot(server)
if err != nil {
t.Fatalf("GetServerDiagnosticsSnapshot failed: %v", err)
}
if got, want := len(serverSnapshot.Records), 1; got != want {
t.Fatalf("server record snapshot count = %d, want %d", got, want)
}
if got, want := serverSnapshot.Summary.RecordCount, 1; got != want {
t.Fatalf("server RecordCount = %d, want %d", got, want)
}
if got, want := serverSnapshot.Summary.ActiveRecordCount, 1; got != want {
t.Fatalf("server ActiveRecordCount = %d, want %d", got, want)
}
serverRecord := serverSnapshot.Records[0]
if got := serverRecord.BatchFramesReceived; got < 1 {
t.Fatalf("server BatchFramesReceived = %d, want >= 1", got)
}
if got := serverRecord.AckFramesSent; got < 1 {
t.Fatalf("server AckFramesSent = %d, want >= 1", got)
}
if got := serverSnapshot.Summary.RecordTelemetry.FrameReceiveCount; got < 1 {
t.Fatalf("server RecordTelemetry.FrameReceiveCount = %d, want >= 1", got)
}
close(recordReleaseCh)
select {
case err := <-recordHandlerDone:
if err != nil {
t.Fatalf("record handler failed: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting record handler completion")
}
_ = record.Close()
}
func TestGetServerDiagnosticsSnapshotAggregatesStaleAndResetState(t *testing.T) { func TestGetServerDiagnosticsSnapshotAggregatesStaleAndResetState(t *testing.T) {
server := NewServer().(*ServerCommon) server := NewServer().(*ServerCommon)
@ -323,6 +454,127 @@ func TestDiagnosticsSummaryClassifiesResetCauses(t *testing.T) {
} }
} }
func TestDiagnosticsSummaryAggregatesRecordTelemetry(t *testing.T) {
summary := summarizeClientDiagnosticsSnapshot(ClientDiagnosticsSnapshot{
Records: []RecordSnapshot{
{
ID: "record-active",
BindingCurrent: true,
TransportAttached: true,
TransportCurrent: true,
OutstandingRecords: 3,
OutstandingBytes: 4096,
PendingApplyRecords: 2,
PendingAckRecords: 1,
PeakPendingApplyRecords: 5,
BatchFramesSent: 10,
AckFramesSent: 4,
ErrorFramesSent: 1,
BatchFramesReceived: 8,
AckFramesReceived: 3,
ErrorFramesReceived: 0,
PiggybackAckSent: 6,
PiggybackAckReceived: 2,
BarrierCount: 4,
BarrierFlushWaitDuration: 10 * time.Millisecond,
BarrierApplyWaitDuration: 30 * time.Millisecond,
},
{
ID: "record-reset",
ResetError: errTransportDetached.Error(),
OutstandingRecords: 1,
OutstandingBytes: 512,
PendingApplyRecords: 3,
PendingAckRecords: 2,
PeakPendingApplyRecords: 7,
BatchFramesSent: 2,
AckFramesSent: 1,
ErrorFramesSent: 1,
BatchFramesReceived: 1,
AckFramesReceived: 1,
ErrorFramesReceived: 1,
PiggybackAckSent: 1,
PiggybackAckReceived: 1,
BarrierCount: 1,
BarrierFlushWaitDuration: 5 * time.Millisecond,
BarrierApplyWaitDuration: 15 * time.Millisecond,
},
},
})
if got, want := summary.RecordCount, 2; got != want {
t.Fatalf("RecordCount = %d, want %d", got, want)
}
if got, want := summary.ActiveRecordCount, 1; got != want {
t.Fatalf("ActiveRecordCount = %d, want %d", got, want)
}
if got, want := summary.ResetRecordCount, 1; got != want {
t.Fatalf("ResetRecordCount = %d, want %d", got, want)
}
if got, want := summary.RecordResetCauses.Total, 1; got != want {
t.Fatalf("RecordResetCauses.Total = %d, want %d", got, want)
}
if got, want := summary.RecordResetCauses.TransportDetached, 1; got != want {
t.Fatalf("RecordResetCauses.TransportDetached = %d, want %d", got, want)
}
telemetry := summary.RecordTelemetry
if got, want := telemetry.BatchFramesSent, int64(12); got != want {
t.Fatalf("BatchFramesSent = %d, want %d", got, want)
}
if got, want := telemetry.AckFramesSent, int64(5); got != want {
t.Fatalf("AckFramesSent = %d, want %d", got, want)
}
if got, want := telemetry.ErrorFramesSent, int64(2); got != want {
t.Fatalf("ErrorFramesSent = %d, want %d", got, want)
}
if got, want := telemetry.BatchFramesReceived, int64(9); got != want {
t.Fatalf("BatchFramesReceived = %d, want %d", got, want)
}
if got, want := telemetry.AckFramesReceived, int64(4); got != want {
t.Fatalf("AckFramesReceived = %d, want %d", got, want)
}
if got, want := telemetry.ErrorFramesReceived, int64(1); got != want {
t.Fatalf("ErrorFramesReceived = %d, want %d", got, want)
}
if got, want := telemetry.FrameSendCount, int64(19); got != want {
t.Fatalf("FrameSendCount = %d, want %d", got, want)
}
if got, want := telemetry.FrameReceiveCount, int64(14); got != want {
t.Fatalf("FrameReceiveCount = %d, want %d", got, want)
}
if got, want := telemetry.PiggybackAckSent, int64(7); got != want {
t.Fatalf("PiggybackAckSent = %d, want %d", got, want)
}
if got, want := telemetry.PiggybackAckReceived, int64(3); got != want {
t.Fatalf("PiggybackAckReceived = %d, want %d", got, want)
}
if got, want := telemetry.BarrierCount, int64(5); got != want {
t.Fatalf("BarrierCount = %d, want %d", got, want)
}
if got, want := telemetry.BarrierFlushWaitDuration, 15*time.Millisecond; got != want {
t.Fatalf("BarrierFlushWaitDuration = %v, want %v", got, want)
}
if got, want := telemetry.BarrierApplyWaitDuration, 45*time.Millisecond; got != want {
t.Fatalf("BarrierApplyWaitDuration = %v, want %v", got, want)
}
if got, want := telemetry.OutstandingRecords, 4; got != want {
t.Fatalf("OutstandingRecords = %d, want %d", got, want)
}
if got, want := telemetry.OutstandingBytes, 4608; got != want {
t.Fatalf("OutstandingBytes = %d, want %d", got, want)
}
if got, want := telemetry.PendingApplyRecords, 5; got != want {
t.Fatalf("PendingApplyRecords = %d, want %d", got, want)
}
if got, want := telemetry.PendingAckRecords, 3; got != want {
t.Fatalf("PendingAckRecords = %d, want %d", got, want)
}
if got, want := telemetry.PeakPendingApplyRecords, 7; got != want {
t.Fatalf("PeakPendingApplyRecords = %d, want %d", got, want)
}
}
func TestDiagnosticsSummaryAggregatesTransferTelemetry(t *testing.T) { func TestDiagnosticsSummaryAggregatesTransferTelemetry(t *testing.T) {
summary := summarizeClientDiagnosticsSnapshot(ClientDiagnosticsSnapshot{ summary := summarizeClientDiagnosticsSnapshot(ClientDiagnosticsSnapshot{
Transfers: []TransferSnapshot{ Transfers: []TransferSnapshot{

View File

@ -7,12 +7,14 @@ import (
const ( const (
recordFrameMagic = "NRS1" recordFrameMagic = "NRS1"
recordFrameVersion = 1 recordFrameVersionV1 = 1
recordFrameVersionV2 = 2
recordFrameTypeBatch uint8 = 1 recordFrameTypeBatch uint8 = 1
recordFrameTypeAck uint8 = 2 recordFrameTypeAck uint8 = 2
recordFrameTypeError uint8 = 3 recordFrameTypeError uint8 = 3
recordFrameHeaderSize = 8 recordFrameHeaderSize = 8
recordBatchHeaderSize = 10 recordBatchHeaderV1Size = 10
recordBatchHeaderV2Size = 18
recordErrorHeaderSize = 16 recordErrorHeaderSize = 16
) )
@ -27,6 +29,7 @@ type recordOutboundMessage struct {
} }
type recordFrame struct { type recordFrame struct {
Version uint8
Type uint8 Type uint8
Batch []recordOutboundMessage Batch []recordOutboundMessage
AckSeq uint64 AckSeq uint64
@ -34,7 +37,7 @@ type recordFrame struct {
Retryable bool Retryable bool
} }
func encodeRecordBatchFrame(batch []recordOutboundMessage) ([]byte, error) { func encodeRecordBatchFrame(batch []recordOutboundMessage, ackSeq uint64, useV2 bool) ([]byte, error) {
if len(batch) == 0 { if len(batch) == 0 {
return nil, nil return nil, nil
} }
@ -42,7 +45,13 @@ func encodeRecordBatchFrame(batch []recordOutboundMessage) ([]byte, error) {
if firstSeq == 0 { if firstSeq == 0 {
return nil, errRecordSeqInvalid return nil, errRecordSeqInvalid
} }
size := recordFrameHeaderSize + recordBatchHeaderSize version := uint8(recordFrameVersionV1)
batchHeaderSize := recordBatchHeaderV1Size
if useV2 {
version = recordFrameVersionV2
batchHeaderSize = recordBatchHeaderV2Size
}
size := recordFrameHeaderSize + batchHeaderSize
for index, item := range batch { for index, item := range batch {
wantSeq := firstSeq + uint64(index) wantSeq := firstSeq + uint64(index)
if item.Seq != wantSeq { if item.Seq != wantSeq {
@ -52,11 +61,14 @@ func encodeRecordBatchFrame(batch []recordOutboundMessage) ([]byte, error) {
} }
frame := make([]byte, size) frame := make([]byte, size)
copy(frame[:4], recordFrameMagic) copy(frame[:4], recordFrameMagic)
frame[4] = recordFrameVersion frame[4] = version
frame[5] = recordFrameTypeBatch frame[5] = recordFrameTypeBatch
binary.BigEndian.PutUint16(frame[8:10], uint16(len(batch))) binary.BigEndian.PutUint16(frame[8:10], uint16(len(batch)))
binary.BigEndian.PutUint64(frame[10:18], firstSeq) binary.BigEndian.PutUint64(frame[10:18], firstSeq)
offset := recordFrameHeaderSize + recordBatchHeaderSize offset := recordFrameHeaderSize + batchHeaderSize
if useV2 {
binary.BigEndian.PutUint64(frame[18:26], ackSeq)
}
for _, item := range batch { for _, item := range batch {
binary.BigEndian.PutUint32(frame[offset:offset+4], uint32(len(item.Payload))) binary.BigEndian.PutUint32(frame[offset:offset+4], uint32(len(item.Payload)))
offset += 4 offset += 4
@ -69,7 +81,7 @@ func encodeRecordBatchFrame(batch []recordOutboundMessage) ([]byte, error) {
func encodeRecordAckFrame(ackSeq uint64) ([]byte, error) { func encodeRecordAckFrame(ackSeq uint64) ([]byte, error) {
frame := make([]byte, recordFrameHeaderSize+8) frame := make([]byte, recordFrameHeaderSize+8)
copy(frame[:4], recordFrameMagic) copy(frame[:4], recordFrameMagic)
frame[4] = recordFrameVersion frame[4] = recordFrameVersionV1
frame[5] = recordFrameTypeAck frame[5] = recordFrameTypeAck
binary.BigEndian.PutUint64(frame[8:16], ackSeq) binary.BigEndian.PutUint64(frame[8:16], ackSeq)
return frame, nil return frame, nil
@ -83,7 +95,7 @@ func encodeRecordErrorFrame(failure RecordFailure) ([]byte, error) {
msgBytes := []byte(failure.Message) msgBytes := []byte(failure.Message)
frame := make([]byte, recordFrameHeaderSize+recordErrorHeaderSize+len(codeBytes)+len(msgBytes)) frame := make([]byte, recordFrameHeaderSize+recordErrorHeaderSize+len(codeBytes)+len(msgBytes))
copy(frame[:4], recordFrameMagic) copy(frame[:4], recordFrameMagic)
frame[4] = recordFrameVersion frame[4] = recordFrameVersionV1
frame[5] = recordFrameTypeError frame[5] = recordFrameTypeError
if failure.Retryable { if failure.Retryable {
frame[6] = 1 frame[6] = 1
@ -102,30 +114,62 @@ func decodeRecordFrame(payload []byte) (recordFrame, error) {
if len(payload) < recordFrameHeaderSize || string(payload[:4]) != recordFrameMagic { if len(payload) < recordFrameHeaderSize || string(payload[:4]) != recordFrameMagic {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
if payload[4] != recordFrameVersion { version := payload[4]
return recordFrame{}, errRecordFrameInvalid
}
frameType := payload[5] frameType := payload[5]
switch version {
case recordFrameVersionV1:
switch frameType { switch frameType {
case recordFrameTypeBatch: case recordFrameTypeBatch:
return decodeRecordBatchFrame(payload) return decodeRecordBatchFrameV1(payload)
case recordFrameTypeAck: case recordFrameTypeAck:
if len(payload) != recordFrameHeaderSize+8 { if len(payload) != recordFrameHeaderSize+8 {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
return recordFrame{ return recordFrame{
Version: recordFrameVersionV1,
Type: recordFrameTypeAck, Type: recordFrameTypeAck,
AckSeq: binary.BigEndian.Uint64(payload[8:16]), AckSeq: binary.BigEndian.Uint64(payload[8:16]),
}, nil }, nil
case recordFrameTypeError: case recordFrameTypeError:
return decodeRecordErrorFrame(payload) frame, err := decodeRecordErrorFrame(payload)
if err != nil {
return recordFrame{}, err
}
frame.Version = recordFrameVersionV1
return frame, nil
default:
return recordFrame{}, errRecordFrameInvalid
}
case recordFrameVersionV2:
switch frameType {
case recordFrameTypeBatch:
return decodeRecordBatchFrameV2(payload)
case recordFrameTypeAck:
if len(payload) != recordFrameHeaderSize+8 {
return recordFrame{}, errRecordFrameInvalid
}
return recordFrame{
Version: recordFrameVersionV2,
Type: recordFrameTypeAck,
AckSeq: binary.BigEndian.Uint64(payload[8:16]),
}, nil
case recordFrameTypeError:
frame, err := decodeRecordErrorFrame(payload)
if err != nil {
return recordFrame{}, err
}
frame.Version = recordFrameVersionV2
return frame, nil
default:
return recordFrame{}, errRecordFrameInvalid
}
default: default:
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
} }
func decodeRecordBatchFrame(payload []byte) (recordFrame, error) { func decodeRecordBatchFrameV1(payload []byte) (recordFrame, error) {
if len(payload) < recordFrameHeaderSize+recordBatchHeaderSize { if len(payload) < recordFrameHeaderSize+recordBatchHeaderV1Size {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
count := int(binary.BigEndian.Uint16(payload[8:10])) count := int(binary.BigEndian.Uint16(payload[8:10]))
@ -133,7 +177,7 @@ func decodeRecordBatchFrame(payload []byte) (recordFrame, error) {
if count <= 0 || firstSeq == 0 { if count <= 0 || firstSeq == 0 {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
offset := recordFrameHeaderSize + recordBatchHeaderSize offset := recordFrameHeaderSize + recordBatchHeaderV1Size
batch := make([]recordOutboundMessage, 0, count) batch := make([]recordOutboundMessage, 0, count)
for index := 0; index < count; index++ { for index := 0; index < count; index++ {
if offset+4 > len(payload) { if offset+4 > len(payload) {
@ -155,11 +199,51 @@ func decodeRecordBatchFrame(payload []byte) (recordFrame, error) {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid
} }
return recordFrame{ return recordFrame{
Version: recordFrameVersionV1,
Type: recordFrameTypeBatch, Type: recordFrameTypeBatch,
Batch: batch, Batch: batch,
}, nil }, nil
} }
func decodeRecordBatchFrameV2(payload []byte) (recordFrame, error) {
if len(payload) < recordFrameHeaderSize+recordBatchHeaderV2Size {
return recordFrame{}, errRecordFrameInvalid
}
count := int(binary.BigEndian.Uint16(payload[8:10]))
firstSeq := binary.BigEndian.Uint64(payload[10:18])
ackSeq := binary.BigEndian.Uint64(payload[18:26])
if count <= 0 || firstSeq == 0 {
return recordFrame{}, errRecordFrameInvalid
}
offset := recordFrameHeaderSize + recordBatchHeaderV2Size
batch := make([]recordOutboundMessage, 0, count)
for index := 0; index < count; index++ {
if offset+4 > len(payload) {
return recordFrame{}, errRecordFrameInvalid
}
itemLen := int(binary.BigEndian.Uint32(payload[offset : offset+4]))
offset += 4
if itemLen < 0 || offset+itemLen > len(payload) {
return recordFrame{}, errRecordFrameInvalid
}
item := recordOutboundMessage{
Seq: firstSeq + uint64(index),
Payload: append([]byte(nil), payload[offset:offset+itemLen]...),
}
offset += itemLen
batch = append(batch, item)
}
if offset != len(payload) {
return recordFrame{}, errRecordFrameInvalid
}
return recordFrame{
Version: recordFrameVersionV2,
Type: recordFrameTypeBatch,
Batch: batch,
AckSeq: ackSeq,
}, nil
}
func decodeRecordErrorFrame(payload []byte) (recordFrame, error) { func decodeRecordErrorFrame(payload []byte) (recordFrame, error) {
if len(payload) < recordFrameHeaderSize+recordErrorHeaderSize { if len(payload) < recordFrameHeaderSize+recordErrorHeaderSize {
return recordFrame{}, errRecordFrameInvalid return recordFrame{}, errRecordFrameInvalid

73
record_codec_test.go Normal file
View File

@ -0,0 +1,73 @@
package notify
import "testing"
func TestEncodeDecodeRecordBatchFrameV1(t *testing.T) {
batch := []recordOutboundMessage{
{Seq: 7, Payload: []byte("alpha")},
{Seq: 8, Payload: []byte("beta")},
}
payload, err := encodeRecordBatchFrame(batch, 0, false)
if err != nil {
t.Fatalf("encodeRecordBatchFrame v1 failed: %v", err)
}
frame, err := decodeRecordFrame(payload)
if err != nil {
t.Fatalf("decodeRecordFrame v1 failed: %v", err)
}
if got, want := frame.Version, uint8(recordFrameVersionV1); got != want {
t.Fatalf("frame version = %d, want %d", got, want)
}
if got, want := frame.Type, recordFrameTypeBatch; got != want {
t.Fatalf("frame type = %d, want %d", got, want)
}
if frame.AckSeq != 0 {
t.Fatalf("frame ack seq = %d, want 0", frame.AckSeq)
}
if got, want := len(frame.Batch), len(batch); got != want {
t.Fatalf("batch len = %d, want %d", got, want)
}
for i := range batch {
if got, want := frame.Batch[i].Seq, batch[i].Seq; got != want {
t.Fatalf("batch[%d].seq = %d, want %d", i, got, want)
}
if got, want := string(frame.Batch[i].Payload), string(batch[i].Payload); got != want {
t.Fatalf("batch[%d].payload = %q, want %q", i, got, want)
}
}
}
func TestEncodeDecodeRecordBatchFrameV2CarriesAckSeq(t *testing.T) {
batch := []recordOutboundMessage{
{Seq: 11, Payload: []byte("alpha")},
{Seq: 12, Payload: []byte("beta")},
}
payload, err := encodeRecordBatchFrame(batch, 9, true)
if err != nil {
t.Fatalf("encodeRecordBatchFrame v2 failed: %v", err)
}
frame, err := decodeRecordFrame(payload)
if err != nil {
t.Fatalf("decodeRecordFrame v2 failed: %v", err)
}
if got, want := frame.Version, uint8(recordFrameVersionV2); got != want {
t.Fatalf("frame version = %d, want %d", got, want)
}
if got, want := frame.Type, recordFrameTypeBatch; got != want {
t.Fatalf("frame type = %d, want %d", got, want)
}
if got, want := frame.AckSeq, uint64(9); got != want {
t.Fatalf("frame ack seq = %d, want %d", got, want)
}
if got, want := len(frame.Batch), len(batch); got != want {
t.Fatalf("batch len = %d, want %d", got, want)
}
for i := range batch {
if got, want := frame.Batch[i].Seq, batch[i].Seq; got != want {
t.Fatalf("batch[%d].seq = %d, want %d", i, got, want)
}
if got, want := string(frame.Batch[i].Payload), string(batch[i].Payload); got != want {
t.Fatalf("batch[%d].payload = %q, want %q", i, got, want)
}
}
}

48
record_negotiation.go Normal file
View File

@ -0,0 +1,48 @@
package notify
const (
recordStreamMetadataCapBatchAckKey = "_notify.record_cap_batch_ack"
recordStreamMetadataUseBatchAckKey = "_notify.record_use_batch_ack"
recordStreamMetadataEnabledValue = "1"
)
func advertiseRecordStreamOpenMetadata(metadata StreamMetadata) StreamMetadata {
metadata = cloneStreamMetadata(metadata)
if metadata == nil {
metadata = make(StreamMetadata, 1)
}
metadata[recordStreamMetadataCapBatchAckKey] = recordStreamMetadataEnabledValue
return metadata
}
func negotiateRecordStreamOpenMetadata(channel StreamChannel, metadata StreamMetadata) (StreamMetadata, StreamMetadata) {
metadata = cloneStreamMetadata(metadata)
if normalizeStreamChannel(channel) != StreamRecordChannel {
return metadata, nil
}
if metadata[recordStreamMetadataCapBatchAckKey] != recordStreamMetadataEnabledValue {
return metadata, nil
}
metadata[recordStreamMetadataUseBatchAckKey] = recordStreamMetadataEnabledValue
return metadata, StreamMetadata{
recordStreamMetadataUseBatchAckKey: recordStreamMetadataEnabledValue,
}
}
func mergeStreamMetadata(base StreamMetadata, overlay StreamMetadata) StreamMetadata {
if len(base) == 0 && len(overlay) == 0 {
return nil
}
merged := cloneStreamMetadata(base)
if merged == nil {
merged = make(StreamMetadata, len(overlay))
}
for key, value := range overlay {
merged[key] = value
}
return merged
}
func recordStreamUseBatchAck(metadata StreamMetadata) bool {
return metadata[recordStreamMetadataUseBatchAckKey] == recordStreamMetadataEnabledValue
}

View File

@ -0,0 +1,78 @@
package notify
import (
"context"
"net"
"testing"
"time"
)
func TestNegotiateRecordStreamOpenMetadataEnablesBatchAck(t *testing.T) {
reqMetadata, respMetadata := negotiateRecordStreamOpenMetadata(StreamRecordChannel, StreamMetadata{
recordStreamMetadataCapBatchAckKey: recordStreamMetadataEnabledValue,
})
if !recordStreamUseBatchAck(reqMetadata) {
t.Fatal("request metadata should enable batch ack")
}
if !recordStreamUseBatchAck(respMetadata) {
t.Fatal("response metadata should enable batch ack")
}
}
func TestNegotiateRecordStreamOpenMetadataKeepsFallbackWithoutCapability(t *testing.T) {
reqMetadata, respMetadata := negotiateRecordStreamOpenMetadata(StreamRecordChannel, nil)
if recordStreamUseBatchAck(reqMetadata) {
t.Fatalf("request metadata should keep fallback mode: %+v", reqMetadata)
}
if recordStreamUseBatchAck(respMetadata) {
t.Fatalf("response metadata should keep fallback mode: %+v", respMetadata)
}
}
func TestOpenRecordStreamNegotiatesBatchAck(t *testing.T) {
server := NewServer().(*ServerCommon)
secret := []byte("0123456789abcdef0123456789abcdef")
server = newRunningPeerAttachServerForTest(t, func(server *ServerCommon) {
server.SetSecretKey(secret)
})
acceptedCh := make(chan RecordAcceptInfo, 1)
server.SetRecordStreamHandler(func(info RecordAcceptInfo) error {
acceptedCh <- info
return nil
})
client := NewClient().(*ClientCommon)
client.SetSecretKey(secret)
left, right := net.Pipe()
defer right.Close()
bootstrapPeerAttachConnForTest(t, server, right)
if err := client.ConnectByConn(left); err != nil {
t.Fatalf("client ConnectByConn failed: %v", err)
}
defer func() {
client.setByeFromServer(true)
_ = client.Stop()
}()
record, err := client.OpenRecordStream(context.Background(), RecordOpenOptions{})
if err != nil {
t.Fatalf("OpenRecordStream failed: %v", err)
}
defer func() {
_ = record.Close()
}()
if !recordStreamUseBatchAck(record.Metadata()) {
t.Fatalf("client record stream metadata should negotiate batch ack: %+v", record.Metadata())
}
select {
case accepted := <-acceptedCh:
if !recordStreamUseBatchAck(accepted.Metadata) {
t.Fatalf("accepted record metadata should negotiate batch ack: %+v", accepted.Metadata)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting accepted record stream")
}
}

View File

@ -1,14 +1,20 @@
package notify package notify
import "sync" import (
"strconv"
"sync"
)
type recordRuntime struct { type recordRuntime struct {
mu sync.RWMutex mu sync.RWMutex
handler func(RecordAcceptInfo) error handler func(RecordAcceptInfo) error
records map[string]*recordStream
} }
func newRecordRuntime() *recordRuntime { func newRecordRuntime() *recordRuntime {
return &recordRuntime{} return &recordRuntime{
records: make(map[string]*recordStream),
}
} }
func (r *recordRuntime) setHandler(fn func(RecordAcceptInfo) error) { func (r *recordRuntime) setHandler(fn func(RecordAcceptInfo) error) {
@ -42,3 +48,120 @@ func (s *ServerCommon) getRecordRuntime() *recordRuntime {
} }
return s.recordRuntime return s.recordRuntime
} }
func (r *recordRuntime) register(record *recordStream) {
if r == nil || record == nil {
return
}
key := record.runtimeRegistryKey()
if key == "" {
return
}
r.mu.Lock()
r.records[key] = record
r.mu.Unlock()
}
func (r *recordRuntime) remove(key string) {
if r == nil || key == "" {
return
}
r.mu.Lock()
delete(r.records, key)
r.mu.Unlock()
}
func (r *recordRuntime) snapshots() []RecordSnapshot {
if r == nil {
return nil
}
r.mu.RLock()
records := make([]*recordStream, 0, len(r.records))
for _, record := range r.records {
if record == nil {
continue
}
records = append(records, record)
}
r.mu.RUnlock()
snapshots := make([]RecordSnapshot, 0, len(records))
for _, record := range records {
snapshots = append(snapshots, record.snapshot())
}
sortRecordSnapshots(snapshots)
return snapshots
}
func bindRecordRuntime(record RecordStream, runtime *recordRuntime) {
if runtime == nil || record == nil {
return
}
rs, ok := record.(*recordStream)
if !ok {
return
}
rs.bindRuntime(runtime)
}
func (r *recordStream) bindRuntime(runtime *recordRuntime) {
if r == nil || runtime == nil {
return
}
key := r.runtimeRegistryKey()
if key == "" {
return
}
r.mu.Lock()
r.runtime = runtime
r.runtimeKey = key
r.mu.Unlock()
runtime.register(r)
r.runtimeWatchOnce.Do(func() {
go func() {
streamCtx := r.stream.Context()
if streamCtx == nil {
<-r.ctx.Done()
} else {
select {
case <-r.ctx.Done():
case <-streamCtx.Done():
}
}
r.detachRuntime()
}()
})
}
func (r *recordStream) detachRuntime() {
if r == nil {
return
}
r.runtimeDetachOnce.Do(func() {
r.mu.Lock()
runtime := r.runtime
key := r.runtimeKey
r.runtime = nil
r.runtimeKey = ""
r.mu.Unlock()
if runtime != nil {
runtime.remove(key)
}
})
}
func (r *recordStream) runtimeRegistryKey() string {
if r == nil || r.stream == nil {
return ""
}
scope := ""
dataID := uint64(0)
if stream, ok := r.stream.(*streamHandle); ok {
scope = normalizeFileScope(stream.runtimeScope)
dataID = stream.dataID
}
key := scope + "\x00" + r.stream.ID()
if dataID != 0 {
key += "\x01" + strconv.FormatUint(dataID, 10)
}
return key
}

252
record_snapshot.go Normal file
View File

@ -0,0 +1,252 @@
package notify
import (
"errors"
"io"
"sort"
"time"
)
type RecordSnapshot struct {
ID string
DataID uint64
Scope string
Metadata StreamMetadata
UseBatchAck bool
BindingOwner string
BindingAlive bool
BindingCurrent bool
BindingReason string
BindingError string
SessionEpoch uint64
LogicalClientID string
LocalAddress string
RemoteAddress string
TransportGeneration uint64
TransportAttached bool
TransportHasRuntimeConn bool
TransportCurrent bool
TransportDetachReason string
TransportDetachKind string
TransportDetachGeneration uint64
TransportDetachError string
TransportDetachedAt time.Time
ReattachEligible bool
LocalClosed bool
LocalReadClosed bool
RemoteClosed bool
PeerReadClosed bool
OutboundClosed bool
NextOutboundSeq uint64
EnqueuedOutboundSeq uint64
FlushedOutboundSeq uint64
AckedOutboundSeq uint64
OutstandingRecords int
OutstandingBytes int
InboundReceivedSeq uint64
InboundAppliedSeq uint64
InboundAckSentSeq uint64
PendingApplyRecords int
PendingAckRecords int
PeakPendingApplyRecords int
BatchFramesSent int64
AckFramesSent int64
ErrorFramesSent int64
BatchFramesReceived int64
AckFramesReceived int64
ErrorFramesReceived int64
PiggybackAckSent int64
PiggybackAckReceived int64
BarrierCount int64
BarrierFlushWaitDuration time.Duration
BarrierApplyWaitDuration time.Duration
OpenedAt time.Time
LastReadAt time.Time
LastWriteAt time.Time
StreamResetError string
ReadError string
TerminalError string
ResetError string
}
type clientRecordSnapshotReader interface {
clientRecordSnapshots() []RecordSnapshot
}
type serverRecordSnapshotReader interface {
serverRecordSnapshots() []RecordSnapshot
}
var (
errClientRecordSnapshotNil = errors.New("client record snapshot target is nil")
errServerRecordSnapshotNil = errors.New("server record snapshot target is nil")
errClientRecordSnapshotUnsupported = errors.New("client record snapshot target type is unsupported")
errServerRecordSnapshotUnsupported = errors.New("server record snapshot target type is unsupported")
)
func GetClientRecordSnapshots(c Client) ([]RecordSnapshot, error) {
if c == nil {
return nil, errClientRecordSnapshotNil
}
reader, ok := any(c).(clientRecordSnapshotReader)
if !ok {
return nil, errClientRecordSnapshotUnsupported
}
return reader.clientRecordSnapshots(), nil
}
func GetServerRecordSnapshots(s Server) ([]RecordSnapshot, error) {
if s == nil {
return nil, errServerRecordSnapshotNil
}
reader, ok := any(s).(serverRecordSnapshotReader)
if !ok {
return nil, errServerRecordSnapshotUnsupported
}
return reader.serverRecordSnapshots(), nil
}
func (c *ClientCommon) clientRecordSnapshots() []RecordSnapshot {
return recordSnapshotsFromRuntime(c.getRecordRuntime())
}
func (s *ServerCommon) serverRecordSnapshots() []RecordSnapshot {
return recordSnapshotsFromRuntime(s.getRecordRuntime())
}
func recordSnapshotsFromRuntime(runtime *recordRuntime) []RecordSnapshot {
if runtime == nil {
return nil
}
return runtime.snapshots()
}
func sortRecordSnapshots(src []RecordSnapshot) {
sort.Slice(src, func(i, j int) bool {
if src[i].Scope != src[j].Scope {
return src[i].Scope < src[j].Scope
}
if src[i].ID != src[j].ID {
return src[i].ID < src[j].ID
}
if src[i].DataID != src[j].DataID {
return src[i].DataID < src[j].DataID
}
return src[i].TransportGeneration < src[j].TransportGeneration
})
}
func (r *recordStream) snapshot() RecordSnapshot {
if r == nil {
return RecordSnapshot{}
}
snapshot := RecordSnapshot{}
if stream, ok := r.stream.(*streamHandle); ok {
snapshot = recordSnapshotFromStreamSnapshot(stream.snapshot())
} else if r.stream != nil {
snapshot.ID = r.stream.ID()
snapshot.Metadata = cloneStreamMetadata(r.stream.Metadata())
snapshot.TransportGeneration = r.stream.TransportGeneration()
if addr := r.stream.LocalAddr(); addr != nil {
snapshot.LocalAddress = addr.String()
}
if addr := r.stream.RemoteAddr(); addr != nil {
snapshot.RemoteAddress = addr.String()
}
if logical := r.stream.LogicalConn(); logical != nil {
snapshot.LogicalClientID = logical.ID()
}
}
snapshot.UseBatchAck = r.useBatchAck
snapshot.BatchFramesSent = r.obs.batchFramesSent.Load()
snapshot.AckFramesSent = r.obs.ackFramesSent.Load()
snapshot.ErrorFramesSent = r.obs.errorFramesSent.Load()
snapshot.BatchFramesReceived = r.obs.batchFramesReceived.Load()
snapshot.AckFramesReceived = r.obs.ackFramesReceived.Load()
snapshot.ErrorFramesReceived = r.obs.errorFramesReceived.Load()
snapshot.PiggybackAckSent = r.obs.piggybackAckSent.Load()
snapshot.PiggybackAckReceived = r.obs.piggybackAckReceived.Load()
snapshot.BarrierCount = r.obs.barrierCount.Load()
snapshot.BarrierFlushWaitDuration = time.Duration(r.obs.barrierFlushWaitNanos.Load())
snapshot.BarrierApplyWaitDuration = time.Duration(r.obs.barrierApplyWaitNanos.Load())
r.mu.Lock()
snapshot.OutboundClosed = r.outboundClosed
snapshot.NextOutboundSeq = r.nextOutboundSeq
snapshot.EnqueuedOutboundSeq = r.enqueuedOutboundSeq
snapshot.FlushedOutboundSeq = r.flushedOutboundSeq
snapshot.AckedOutboundSeq = r.ackedOutboundSeq
snapshot.OutstandingRecords = r.outstandingRecords
snapshot.OutstandingBytes = r.outstandingBytes
snapshot.InboundReceivedSeq = r.inboundReceivedSeq
snapshot.InboundAppliedSeq = r.inboundAppliedSeq
snapshot.InboundAckSentSeq = r.inboundAckSentSeq
snapshot.PendingApplyRecords = recordPendingCount(r.inboundReceivedSeq, r.inboundAppliedSeq)
snapshot.PendingAckRecords = recordPendingCount(r.inboundAppliedSeq, r.inboundAckSentSeq)
snapshot.PeakPendingApplyRecords = r.maxPendingApply
if r.readErr != nil && !errors.Is(r.readErr, io.EOF) {
snapshot.ReadError = r.readErr.Error()
}
if r.terminalErr != nil {
snapshot.TerminalError = r.terminalErr.Error()
}
r.mu.Unlock()
switch {
case snapshot.TerminalError != "":
snapshot.ResetError = snapshot.TerminalError
case snapshot.StreamResetError != "":
snapshot.ResetError = snapshot.StreamResetError
case snapshot.ReadError != "":
snapshot.ResetError = snapshot.ReadError
}
return snapshot
}
func recordSnapshotFromStreamSnapshot(stream StreamSnapshot) RecordSnapshot {
return RecordSnapshot{
ID: stream.ID,
DataID: stream.DataID,
Scope: stream.Scope,
Metadata: cloneStreamMetadata(stream.Metadata),
BindingOwner: stream.BindingOwner,
BindingAlive: stream.BindingAlive,
BindingCurrent: stream.BindingCurrent,
BindingReason: stream.BindingReason,
BindingError: stream.BindingError,
SessionEpoch: stream.SessionEpoch,
LogicalClientID: stream.LogicalClientID,
LocalAddress: stream.LocalAddress,
RemoteAddress: stream.RemoteAddress,
TransportGeneration: stream.TransportGeneration,
TransportAttached: stream.TransportAttached,
TransportHasRuntimeConn: stream.TransportHasRuntimeConn,
TransportCurrent: stream.TransportCurrent,
TransportDetachReason: stream.TransportDetachReason,
TransportDetachKind: stream.TransportDetachKind,
TransportDetachGeneration: stream.TransportDetachGeneration,
TransportDetachError: stream.TransportDetachError,
TransportDetachedAt: stream.TransportDetachedAt,
ReattachEligible: stream.ReattachEligible,
LocalClosed: stream.LocalClosed,
LocalReadClosed: stream.LocalReadClosed,
RemoteClosed: stream.RemoteClosed,
PeerReadClosed: stream.PeerReadClosed,
OpenedAt: stream.OpenedAt,
LastReadAt: stream.LastReadAt,
LastWriteAt: stream.LastWriteAt,
StreamResetError: stream.ResetError,
}
}
func recordPendingCount(high uint64, low uint64) int {
if high <= low {
return 0
}
diff := high - low
maxInt := uint64(^uint(0) >> 1)
if diff > maxInt {
return int(maxInt)
}
return int(diff)
}

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"io" "io"
"sync" "sync"
"sync/atomic"
"time" "time"
) )
@ -103,9 +104,24 @@ type recordConfig struct {
type recordFlushRequest struct { type recordFlushRequest struct {
targetSeq uint64 targetSeq uint64
forceAck bool
done chan error done chan error
} }
type recordObservability struct {
batchFramesSent atomic.Int64
ackFramesSent atomic.Int64
errorFramesSent atomic.Int64
batchFramesReceived atomic.Int64
ackFramesReceived atomic.Int64
errorFramesReceived atomic.Int64
piggybackAckSent atomic.Int64
piggybackAckReceived atomic.Int64
barrierCount atomic.Int64
barrierFlushWaitNanos atomic.Int64
barrierApplyWaitNanos atomic.Int64
}
type recordStream struct { type recordStream struct {
stream Stream stream Stream
ctx context.Context ctx context.Context
@ -117,11 +133,18 @@ type recordStream struct {
recvCh chan RecordMessage recvCh chan RecordMessage
ackCh chan struct{} ackCh chan struct{}
readerCh chan struct{} readerCh chan struct{}
useBatchAck bool
obs recordObservability
mu sync.Mutex mu sync.Mutex
stateNotify chan struct{} stateNotify chan struct{}
runtime *recordRuntime
runtimeKey string
runtimeWatchOnce sync.Once
runtimeDetachOnce sync.Once
nextOutboundSeq uint64 nextOutboundSeq uint64
enqueuedOutboundSeq uint64 enqueuedOutboundSeq uint64
flushedOutboundSeq uint64 flushedOutboundSeq uint64
@ -135,6 +158,7 @@ type recordStream struct {
inboundAppliedSeq uint64 inboundAppliedSeq uint64
inboundApplied map[uint64]struct{} inboundApplied map[uint64]struct{}
inboundAckSentSeq uint64 inboundAckSentSeq uint64
maxPendingApply int
remoteClosed bool remoteClosed bool
readErr error readErr error
@ -193,6 +217,7 @@ func recordConfigFromOptions(opt RecordOpenOptions) recordConfig {
func normalizeRecordStreamOpenOptions(opt StreamOpenOptions) StreamOpenOptions { func normalizeRecordStreamOpenOptions(opt StreamOpenOptions) StreamOpenOptions {
opt.Channel = StreamRecordChannel opt.Channel = StreamRecordChannel
opt.Metadata = advertiseRecordStreamOpenMetadata(opt.Metadata)
return opt return opt
} }
@ -216,13 +241,13 @@ func WrapStreamAsRecord(stream Stream, opt RecordOpenOptions) (RecordStream, err
recvCh: make(chan RecordMessage, opt.InboundQueueLimit), recvCh: make(chan RecordMessage, opt.InboundQueueLimit),
ackCh: make(chan struct{}, 1), ackCh: make(chan struct{}, 1),
readerCh: make(chan struct{}), readerCh: make(chan struct{}),
useBatchAck: recordStreamUseBatchAck(stream.Metadata()),
stateNotify: make(chan struct{}), stateNotify: make(chan struct{}),
outstandingSizes: make(map[uint64]int), outstandingSizes: make(map[uint64]int),
inboundApplied: make(map[uint64]struct{}), inboundApplied: make(map[uint64]struct{}),
} }
go record.sendLoop() go record.writerLoop()
go record.ackLoop()
go record.readLoop() go record.readLoop()
return record, nil return record, nil
} }
@ -360,13 +385,20 @@ func (r *recordStream) BarrierTo(ctx context.Context, target uint64) (uint64, er
if target > current { if target > current {
return 0, errRecordSeqInvalid return 0, errRecordSeqInvalid
} }
if err := r.Flush(ctx); err != nil { r.obs.barrierCount.Add(1)
flushStart := time.Now()
err := r.Flush(ctx)
r.obs.barrierFlushWaitNanos.Add(time.Since(flushStart).Nanoseconds())
if err != nil {
return 0, err return 0, err
} }
if target == 0 { if target == 0 {
return 0, nil return 0, nil
} }
if err := r.waitAckedAtLeast(ctx, target); err != nil { applyStart := time.Now()
err = r.waitAckedAtLeast(ctx, target)
r.obs.barrierApplyWaitNanos.Add(time.Since(applyStart).Nanoseconds())
if err != nil {
return 0, err return 0, err
} }
return target, nil return target, nil
@ -520,54 +552,118 @@ func (r *recordStream) waitAckedAtLeast(ctx context.Context, target uint64) erro
} }
} }
func (r *recordStream) sendLoop() { func (r *recordStream) writerLoop() {
var ( var (
batch []recordOutboundMessage batch []recordOutboundMessage
batches int batches int
bytes int bytes int
timer *time.Timer batchTimer *time.Timer
timerCh <-chan time.Time batchTimerCh <-chan time.Time
ackTimer *time.Timer
ackTimerCh <-chan time.Time
) )
stopTimer := func() { stopBatchTimer := func() {
if timer == nil { if batchTimer == nil {
return return
} }
if !timer.Stop() { if !batchTimer.Stop() {
select { select {
case <-timer.C: case <-batchTimer.C:
default: default:
} }
} }
timerCh = nil batchTimerCh = nil
} }
flush := func() error { stopAckTimer := func() {
if len(batch) == 0 { if ackTimer == nil {
return
}
if !ackTimer.Stop() {
select {
case <-ackTimer.C:
default:
}
}
ackTimerCh = nil
}
scheduleAck := func(hasPendingBatch bool, force bool) (uint64, bool) {
ackSeq := r.pendingAckSeq()
if ackSeq == 0 {
stopAckTimer()
return 0, false
}
if force {
stopAckTimer()
return ackSeq, true
}
if hasPendingBatch && r.useBatchAck {
stopAckTimer()
return 0, false
}
if r.shouldSendAckNow() || r.cfg.AckDelay <= 0 {
stopAckTimer()
return ackSeq, true
}
if ackTimer == nil {
ackTimer = time.NewTimer(r.cfg.AckDelay)
} else {
ackTimer.Reset(r.cfg.AckDelay)
}
ackTimerCh = ackTimer.C
return 0, false
}
sendStandaloneAck := func(ackSeq uint64) error {
if ackSeq == 0 {
return nil return nil
} }
payload, err := encodeRecordBatchFrame(batch) payload, err := encodeRecordAckFrame(ackSeq)
if err != nil { if err != nil {
return err return err
} }
if err := r.writePayloadFrame(payload); err != nil { if err := r.writePayloadFrame(payload); err != nil {
return err return err
} }
r.obs.ackFramesSent.Add(1)
r.markAckSent(ackSeq)
return nil
}
flushBatch := func() error {
if len(batch) == 0 {
return nil
}
ackSeq := r.pendingAckSeq()
payload, err := encodeRecordBatchFrame(batch, ackSeq, r.useBatchAck)
if err != nil {
return err
}
if err := r.writePayloadFrame(payload); err != nil {
return err
}
r.obs.batchFramesSent.Add(1)
if r.useBatchAck && ackSeq != 0 {
r.obs.piggybackAckSent.Add(1)
r.markAckSent(ackSeq)
}
r.markFlushed(batch[len(batch)-1].Seq) r.markFlushed(batch[len(batch)-1].Seq)
batch = nil batch = nil
batches = 0 batches = 0
bytes = 0 bytes = 0
stopTimer() stopBatchTimer()
if ackSeq, sendNow := scheduleAck(false, false); sendNow {
return sendStandaloneAck(ackSeq)
}
return nil return nil
} }
flushUntil := func(target uint64) error { flushUntil := func(target uint64) error {
for { for {
if target == 0 { if target == 0 {
return flush() return flushBatch()
} }
if r.flushedAtLeast(target) { if r.flushedAtLeast(target) {
return nil return nil
} }
if len(batch) > 0 && batch[len(batch)-1].Seq >= target { if len(batch) > 0 && batch[len(batch)-1].Seq >= target {
if err := flush(); err != nil { if err := flushBatch(); err != nil {
return err return err
} }
if r.flushedAtLeast(target) { if r.flushedAtLeast(target) {
@ -583,7 +679,7 @@ func (r *recordStream) sendLoop() {
batches++ batches++
bytes += len(req.Payload) bytes += len(req.Payload)
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes { if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flush(); err != nil { if err := flushBatch(); err != nil {
return err return err
} }
} }
@ -598,73 +694,55 @@ func (r *recordStream) sendLoop() {
batches++ batches++
bytes += len(req.Payload) bytes += len(req.Payload)
if len(batch) == 1 && r.cfg.MaxBatchDelay > 0 { if len(batch) == 1 && r.cfg.MaxBatchDelay > 0 {
if timer == nil { if batchTimer == nil {
timer = time.NewTimer(r.cfg.MaxBatchDelay) batchTimer = time.NewTimer(r.cfg.MaxBatchDelay)
} else { } else {
timer.Reset(r.cfg.MaxBatchDelay) batchTimer.Reset(r.cfg.MaxBatchDelay)
} }
timerCh = timer.C batchTimerCh = batchTimer.C
} }
if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes { if batches >= r.cfg.MaxBatchRecords || bytes >= r.cfg.MaxBatchBytes {
if err := flush(); err != nil { if err := flushBatch(); err != nil {
r.setTerminalError(err)
return
}
}
case req := <-r.flushCh:
req.done <- flushUntil(req.targetSeq)
case <-timerCh:
if err := flush(); err != nil {
r.setTerminalError(err)
return
}
}
}
}
func (r *recordStream) ackLoop() {
var (
timer *time.Timer
timerCh <-chan time.Time
)
stopTimer := func() {
if timer == nil {
return
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timerCh = nil
}
for {
select {
case <-r.ctx.Done():
return
case <-r.ackCh:
if r.shouldSendAckNow() {
stopTimer()
if err := r.flushAckNow(); err != nil {
r.setTerminalError(err) r.setTerminalError(err)
return return
} }
continue continue
} }
if timer == nil { if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
timer = time.NewTimer(r.cfg.AckDelay) if err := sendStandaloneAck(ackSeq); err != nil {
} else {
timer.Reset(r.cfg.AckDelay)
}
timerCh = timer.C
case <-timerCh:
stopTimer()
if err := r.flushAckNow(); err != nil {
r.setTerminalError(err) r.setTerminalError(err)
return return
} }
} }
case req := <-r.flushCh:
err := flushUntil(req.targetSeq)
if err == nil && req.forceAck {
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
err = sendStandaloneAck(ackSeq)
}
}
req.done <- err
case <-batchTimerCh:
if err := flushBatch(); err != nil {
r.setTerminalError(err)
return
}
case <-r.ackCh:
if ackSeq, sendNow := scheduleAck(len(batch) > 0, false); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
return
}
}
case <-ackTimerCh:
stopAckTimer()
if ackSeq, sendNow := scheduleAck(len(batch) > 0, true); sendNow {
if err := sendStandaloneAck(ackSeq); err != nil {
r.setTerminalError(err)
return
}
}
}
} }
} }
@ -694,6 +772,15 @@ func (r *recordStream) readLoop() {
} }
switch frame.Type { switch frame.Type {
case recordFrameTypeBatch: case recordFrameTypeBatch:
r.obs.batchFramesReceived.Add(1)
if frame.AckSeq != 0 {
r.obs.piggybackAckReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err)
_ = r.stream.Reset(err)
return
}
}
if err := r.handleBatchFrame(frame.Batch); err != nil { if err := r.handleBatchFrame(frame.Batch); err != nil {
_ = r.sendFailureFrame(RecordFailure{ _ = r.sendFailureFrame(RecordFailure{
FailedSeq: r.nextInboundFailureSeq(), FailedSeq: r.nextInboundFailureSeq(),
@ -705,12 +792,14 @@ func (r *recordStream) readLoop() {
return return
} }
case recordFrameTypeAck: case recordFrameTypeAck:
r.obs.ackFramesReceived.Add(1)
if err := r.handleAckFrame(frame.AckSeq); err != nil { if err := r.handleAckFrame(frame.AckSeq); err != nil {
r.setReadError(err) r.setReadError(err)
_ = r.stream.Reset(err) _ = r.stream.Reset(err)
return return
} }
case recordFrameTypeError: case recordFrameTypeError:
r.obs.errorFramesReceived.Add(1)
r.setReadError(frame.Failure) r.setReadError(frame.Failure)
return return
default: default:
@ -732,6 +821,7 @@ func (r *recordStream) handleBatchFrame(batch []recordOutboundMessage) error {
} }
lastSeq := batch[len(batch)-1].Seq lastSeq := batch[len(batch)-1].Seq
r.inboundReceivedSeq = lastSeq r.inboundReceivedSeq = lastSeq
r.updatePendingApplyLocked()
r.signalStateLocked() r.signalStateLocked()
r.mu.Unlock() r.mu.Unlock()
for _, item := range batch { for _, item := range batch {
@ -889,23 +979,21 @@ func (r *recordStream) shouldSendAckNow() bool {
return r.inboundAppliedSeq > r.inboundAckSentSeq && int(r.inboundAppliedSeq-r.inboundAckSentSeq) >= r.cfg.AckEveryRecords return r.inboundAppliedSeq > r.inboundAckSentSeq && int(r.inboundAppliedSeq-r.inboundAckSentSeq) >= r.cfg.AckEveryRecords
} }
func (r *recordStream) flushAckNow() error { func (r *recordStream) pendingAckSeq() uint64 {
if r == nil { if r == nil {
return errRecordStreamNil return 0
} }
r.mu.Lock() r.mu.Lock()
ackSeq := r.inboundAppliedSeq defer r.mu.Unlock()
if ackSeq <= r.inboundAckSentSeq { if r.inboundAppliedSeq <= r.inboundAckSentSeq {
r.mu.Unlock() return 0
return nil
} }
r.mu.Unlock() return r.inboundAppliedSeq
payload, err := encodeRecordAckFrame(ackSeq) }
if err != nil {
return err func (r *recordStream) markAckSent(ackSeq uint64) {
} if r == nil || ackSeq == 0 {
if err := r.writePayloadFrame(payload); err != nil { return
return err
} }
r.mu.Lock() r.mu.Lock()
if ackSeq > r.inboundAckSentSeq { if ackSeq > r.inboundAckSentSeq {
@ -913,7 +1001,27 @@ func (r *recordStream) flushAckNow() error {
r.signalStateLocked() r.signalStateLocked()
} }
r.mu.Unlock() r.mu.Unlock()
return nil }
func (r *recordStream) flushAckNow() error {
if r == nil {
return errRecordStreamNil
}
req := recordFlushRequest{
forceAck: true,
done: make(chan error, 1),
}
select {
case <-r.ctx.Done():
return r.streamError()
case r.flushCh <- req:
}
select {
case <-r.ctx.Done():
return r.streamError()
case err := <-req.done:
return err
}
} }
func (r *recordStream) sendFailureFrame(failure RecordFailure) error { func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
@ -921,7 +1029,11 @@ func (r *recordStream) sendFailureFrame(failure RecordFailure) error {
if err != nil { if err != nil {
return err return err
} }
return r.writePayloadFrame(payload) if err := r.writePayloadFrame(payload); err != nil {
return err
}
r.obs.errorFramesSent.Add(1)
return nil
} }
func (r *recordStream) writePayloadFrame(payload []byte) error { func (r *recordStream) writePayloadFrame(payload []byte) error {
@ -972,3 +1084,13 @@ func (r *recordStream) signalStateLocked() {
close(r.stateNotify) close(r.stateNotify)
r.stateNotify = make(chan struct{}) r.stateNotify = make(chan struct{})
} }
func (r *recordStream) updatePendingApplyLocked() {
if r == nil {
return
}
pending := recordPendingCount(r.inboundReceivedSeq, r.inboundAppliedSeq)
if pending > r.maxPendingApply {
r.maxPendingApply = pending
}
}

View File

@ -24,6 +24,7 @@ func (s *ServerCommon) OpenRecordStreamLogical(ctx context.Context, logical *Log
_ = stream.Reset(err) _ = stream.Reset(err)
return nil, err return nil, err
} }
bindRecordRuntime(record, s.getRecordRuntime())
return record, nil return record, nil
} }
@ -41,6 +42,7 @@ func (s *ServerCommon) OpenRecordStreamTransport(ctx context.Context, transport
_ = stream.Reset(err) _ = stream.Reset(err)
return nil, err return nil, err
} }
bindRecordRuntime(record, s.getRecordRuntime())
return record, nil return record, nil
} }
@ -68,6 +70,7 @@ func (s *ServerCommon) claimInboundRecordStream(logical *LogicalConn, transport
if err != nil { if err != nil {
return true, err return true, err
} }
bindRecordRuntime(record, runtime)
info := RecordAcceptInfo{ info := RecordAcceptInfo{
ID: stream.ID(), ID: stream.ID(),
Metadata: stream.Metadata(), Metadata: stream.Metadata(),

View File

@ -33,6 +33,7 @@ func (s *ServerCommon) OpenStreamLogical(ctx context.Context, logical *LogicalCo
if resp.DataID != 0 { if resp.DataID != 0 {
req.DataID = resp.DataID req.DataID = resp.DataID
} }
req.Metadata = mergeStreamMetadata(req.Metadata, resp.Metadata)
transport := logical.CurrentTransportConn() transport := logical.CurrentTransportConn()
stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverStreamCloseSender(s, logical, nil), serverStreamResetSender(s, logical, nil), serverStreamDataSender(s, transport), runtime.configSnapshot()) stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverStreamCloseSender(s, logical, nil), serverStreamResetSender(s, logical, nil), serverStreamDataSender(s, transport), runtime.configSnapshot())
if err := runtime.register(scope, stream); err != nil { if err := runtime.register(scope, stream); err != nil {
@ -72,6 +73,7 @@ func (s *ServerCommon) OpenStreamTransport(ctx context.Context, transport *Trans
if resp.DataID != 0 { if resp.DataID != 0 {
req.DataID = resp.DataID req.DataID = resp.DataID
} }
req.Metadata = mergeStreamMetadata(req.Metadata, resp.Metadata)
stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverStreamCloseSender(s, logical, transport), serverStreamResetSender(s, logical, transport), serverStreamDataSender(s, transport), runtime.configSnapshot()) stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, resp.TransportGeneration, serverStreamCloseSender(s, logical, transport), serverStreamResetSender(s, logical, transport), serverStreamDataSender(s, transport), runtime.configSnapshot())
if err := runtime.register(scope, stream); err != nil { if err := runtime.register(scope, stream); err != nil {
_, _ = sendStreamResetServerTransport(context.Background(), s, transport, StreamResetRequest{ _, _ = sendStreamResetServerTransport(context.Background(), s, transport, StreamResetRequest{

View File

@ -20,6 +20,7 @@ type StreamOpenResponse struct {
DataID uint64 DataID uint64
Accepted bool Accepted bool
TransportGeneration uint64 TransportGeneration uint64
Metadata StreamMetadata
Error string Error string
} }
@ -95,6 +96,7 @@ func (c *ClientCommon) handleInboundStreamOpen(msg *Message) {
req.DataID = runtime.nextDataID() req.DataID = runtime.nextDataID()
resp.DataID = req.DataID resp.DataID = req.DataID
} }
req.Metadata, resp.Metadata = negotiateRecordStreamOpenMetadata(req.Channel, req.Metadata)
stream := newStreamHandle(c.clientStopContextSnapshot(), runtime, scope, req, c.currentClientSessionEpoch(), nil, nil, 0, clientStreamCloseSender(c), clientStreamResetSender(c), clientStreamDataSender(c, c.currentClientSessionEpoch()), runtime.configSnapshot()) stream := newStreamHandle(c.clientStopContextSnapshot(), runtime, scope, req, c.currentClientSessionEpoch(), nil, nil, 0, clientStreamCloseSender(c), clientStreamResetSender(c), clientStreamDataSender(c, c.currentClientSessionEpoch()), runtime.configSnapshot())
stream.setClientSnapshotOwner(c) stream.setClientSnapshotOwner(c)
stream.setAddrSnapshot(c.clientStreamAddrSnapshot()) stream.setAddrSnapshot(c.clientStreamAddrSnapshot())
@ -180,6 +182,7 @@ func (s *ServerCommon) handleInboundStreamOpen(msg *Message) {
req.DataID = runtime.nextDataID() req.DataID = runtime.nextDataID()
resp.DataID = req.DataID resp.DataID = req.DataID
} }
req.Metadata, resp.Metadata = negotiateRecordStreamOpenMetadata(req.Channel, req.Metadata)
stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, streamTransportGeneration(logical, transport), serverStreamCloseSender(s, logical, transport), serverStreamResetSender(s, logical, transport), serverStreamDataSender(s, transport), runtime.configSnapshot()) stream := newStreamHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, streamTransportGeneration(logical, transport), serverStreamCloseSender(s, logical, transport), serverStreamResetSender(s, logical, transport), serverStreamDataSender(s, transport), runtime.configSnapshot())
if err := runtime.register(scope, stream); err != nil { if err := runtime.register(scope, stream); err != nil {
resp.Error = err.Error() resp.Error = err.Error()

View File

@ -245,6 +245,10 @@ func (t *TransportConn) runtimeSnapshot() TransportConnRuntimeSnapshot {
snapshot.TransportDetachError = diag.TransportDetachError snapshot.TransportDetachError = diag.TransportDetachError
snapshot.TransportDetachedAt = diag.TransportDetachedAt snapshot.TransportDetachedAt = diag.TransportDetachedAt
snapshot.ReattachEligible = diag.ReattachEligible snapshot.ReattachEligible = diag.ReattachEligible
if snapshot.LogicalAlive && snapshot.TransportDetachReason != "" && !snapshot.Current {
snapshot.LogicalReason = ""
snapshot.LogicalError = ""
}
} }
return snapshot return snapshot
} }