package notify import ( "context" "errors" ) func (s *ServerCommon) SetBulkHandler(fn func(BulkAcceptInfo) error) { runtime := s.getBulkRuntime() if runtime == nil { return } runtime.setHandler(fn) } func (s *ServerCommon) OpenBulkLogical(ctx context.Context, logical *LogicalConn, opt BulkOpenOptions) (Bulk, error) { opt = normalizeBulkOpenOptions(opt) switch opt.Mode { case BulkOpenModeDedicated: opt.Dedicated = true return s.openBulkLogicalWithMode(ctx, logical, opt, false) case BulkOpenModeAuto: if err := logicalDedicatedBulkSupportError(logical); err == nil { dedicatedOpt := opt dedicatedOpt.Mode = BulkOpenModeDedicated dedicatedOpt.Dedicated = true bulk, dedicatedErr := s.openBulkLogicalWithMode(ctx, logical, dedicatedOpt, true) if dedicatedErr == nil { return bulk, nil } sharedOpt := opt sharedOpt.Mode = BulkOpenModeShared sharedOpt.Dedicated = false sharedBulk, sharedErr := s.openBulkLogicalWithMode(ctx, logical, sharedOpt, false) if sharedErr == nil { return sharedBulk, nil } return nil, errors.Join(dedicatedErr, sharedErr) } opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkLogicalWithMode(ctx, logical, opt, false) case BulkOpenModeShared, BulkOpenModeDefault: opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkLogicalWithMode(ctx, logical, opt, false) default: opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkLogicalWithMode(ctx, logical, opt, false) } } func (s *ServerCommon) openBulkLogicalWithMode(ctx context.Context, logical *LogicalConn, opt BulkOpenOptions, waitForReset bool) (Bulk, error) { if s == nil { return nil, errBulkServerNil } opt = applyBulkOpenTuningDefaults(opt, s.bulkOpenTuningSnapshot()) if logical == nil { return nil, errBulkLogicalConnNil } transport := logical.CurrentTransportConn() runtime := s.getBulkRuntime() if runtime == nil { return nil, errBulkRuntimeNil } req := serverBulkRequest(runtime, opt) scope := serverFileScope(logical) if req.Dedicated { if err := logicalDedicatedBulkSupportError(logical); err != nil { return nil, err } } if !validBulkRange(req.Range) { return nil, errBulkRangeInvalid } if _, exists := runtime.lookup(scope, req.BulkID); exists { return nil, errBulkAlreadyExists } if req.DataID == 0 { var reserveErr error req.DataID, reserveErr = runtime.reserveDataID(scope, 0) if reserveErr != nil { return nil, reserveErr } } if req.Dedicated { if req.AttachToken == "" { req.AttachToken = newBulkAttachToken() } bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, logical.transportGenerationSnapshot(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport)) bulk.markAcceptHandled() if err := runtime.adoptReserved(scope, bulk); err != nil { runtime.releaseDataID(scope, req.DataID) return nil, err } s.attachServerDedicatedSidecarIfExists(logical, bulk) resp, err := sendBulkOpenServerTransport(ctx, s, transport, req) if err != nil { runtime.releaseDataID(scope, req.DataID) cleanupErr := s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}, waitForReset) bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } if resp.DataID != 0 && resp.DataID != req.DataID { err = errBulkAlreadyExists cleanupErr := s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{ BulkID: req.BulkID, Error: "bulk dedicated data id mismatch", }, waitForReset) bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } if resp.TransportGeneration != 0 { bulk.setTransportGeneration(resp.TransportGeneration) } if resp.FastPathVersion != 0 { bulk.setFastPathVersion(resp.FastPathVersion) } if resp.AttachToken != "" { bulk.setDedicatedAttachToken(resp.AttachToken) } if err := bulk.waitAcceptReady(ctx); err != nil { var cleanupErr error if bulk.resetErrSnapshot() == nil { cleanupErr = s.cleanupBulkLogicalReset(ctx, logical, transport, BulkResetRequest{ BulkID: req.BulkID, DataID: req.DataID, Error: err.Error(), }, waitForReset) } else { s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{ BulkID: req.BulkID, DataID: req.DataID, Error: err.Error(), }) } bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } return bulk, nil } bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, logical.transportGenerationSnapshot(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport)) bulk.markAcceptHandled() if err := runtime.adoptReserved(scope, bulk); err != nil { runtime.releaseDataID(scope, req.DataID) return nil, err } resp, err := sendBulkOpenServerTransport(ctx, s, transport, req) if err != nil { s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}) bulk.markReset(err) return nil, err } if resp.DataID != 0 && resp.DataID != req.DataID { err = errBulkAlreadyExists s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, Error: "bulk data id mismatch"}) bulk.markReset(err) return nil, err } if resp.FastPathVersion != 0 { bulk.setFastPathVersion(resp.FastPathVersion) } if resp.Dedicated { err = errBulkRejected s.bestEffortBulkResetLogical(logical, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: "shared bulk upgraded to dedicated"}) bulk.markReset(err) return nil, err } if resp.AttachToken != "" { bulk.setDedicatedAttachToken(resp.AttachToken) } if resp.TransportGeneration != 0 { bulk.setTransportGeneration(resp.TransportGeneration) } return bulk, nil } func (s *ServerCommon) OpenBulkTransport(ctx context.Context, transport *TransportConn, opt BulkOpenOptions) (Bulk, error) { opt = normalizeBulkOpenOptions(opt) switch opt.Mode { case BulkOpenModeDedicated: opt.Dedicated = true return s.openBulkTransportWithMode(ctx, transport, opt, false) case BulkOpenModeAuto: if err := transportDedicatedBulkSupportError(transport); err == nil { dedicatedOpt := opt dedicatedOpt.Mode = BulkOpenModeDedicated dedicatedOpt.Dedicated = true bulk, dedicatedErr := s.openBulkTransportWithMode(ctx, transport, dedicatedOpt, true) if dedicatedErr == nil { return bulk, nil } sharedOpt := opt sharedOpt.Mode = BulkOpenModeShared sharedOpt.Dedicated = false sharedBulk, sharedErr := s.openBulkTransportWithMode(ctx, transport, sharedOpt, false) if sharedErr == nil { return sharedBulk, nil } return nil, errors.Join(dedicatedErr, sharedErr) } opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkTransportWithMode(ctx, transport, opt, false) case BulkOpenModeShared, BulkOpenModeDefault: opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkTransportWithMode(ctx, transport, opt, false) default: opt.Mode = BulkOpenModeShared opt.Dedicated = false return s.openBulkTransportWithMode(ctx, transport, opt, false) } } func (s *ServerCommon) openBulkTransportWithMode(ctx context.Context, transport *TransportConn, opt BulkOpenOptions, waitForReset bool) (Bulk, error) { if s == nil { return nil, errBulkServerNil } opt = applyBulkOpenTuningDefaults(opt, s.bulkOpenTuningSnapshot()) if transport == nil { return nil, errBulkTransportNil } logical := transport.LogicalConn() if logical == nil { return nil, errBulkLogicalConnNil } runtime := s.getBulkRuntime() if runtime == nil { return nil, errBulkRuntimeNil } req := serverBulkRequest(runtime, opt) scope := serverFileScope(logical) if req.Dedicated { if err := transportDedicatedBulkSupportError(transport); err != nil { return nil, err } } if !validBulkRange(req.Range) { return nil, errBulkRangeInvalid } if _, exists := runtime.lookup(scope, req.BulkID); exists { return nil, errBulkAlreadyExists } if req.DataID == 0 { var reserveErr error req.DataID, reserveErr = runtime.reserveDataID(scope, 0) if reserveErr != nil { return nil, reserveErr } } if req.Dedicated { if req.AttachToken == "" { req.AttachToken = newBulkAttachToken() } bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, transport.TransportGeneration(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport)) bulk.markAcceptHandled() if err := runtime.adoptReserved(scope, bulk); err != nil { runtime.releaseDataID(scope, req.DataID) return nil, err } s.attachServerDedicatedSidecarIfExists(logical, bulk) resp, err := sendBulkOpenServerTransport(ctx, s, transport, req) if err != nil { runtime.releaseDataID(scope, req.DataID) cleanupErr := s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}, waitForReset) bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } if resp.DataID != 0 && resp.DataID != req.DataID { err = errBulkAlreadyExists cleanupErr := s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{ BulkID: req.BulkID, Error: "bulk dedicated data id mismatch", }, waitForReset) bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } if resp.TransportGeneration != 0 { bulk.setTransportGeneration(resp.TransportGeneration) } if resp.FastPathVersion != 0 { bulk.setFastPathVersion(resp.FastPathVersion) } if resp.AttachToken != "" { bulk.setDedicatedAttachToken(resp.AttachToken) } if err := bulk.waitAcceptReady(ctx); err != nil { var cleanupErr error if bulk.resetErrSnapshot() == nil { cleanupErr = s.cleanupBulkTransportReset(ctx, transport, BulkResetRequest{ BulkID: req.BulkID, DataID: req.DataID, Error: err.Error(), }, waitForReset) } else { s.bestEffortBulkResetTransport(transport, BulkResetRequest{ BulkID: req.BulkID, DataID: req.DataID, Error: err.Error(), }) } bulk.markReset(err) if cleanupErr != nil { return nil, errors.Join(err, cleanupErr) } return nil, err } return bulk, nil } bulk := newBulkHandle(logical.stopContextSnapshot(), runtime, scope, req, 0, logical, transport, transport.TransportGeneration(), serverBulkCloseSender(s, logical, transport), serverBulkResetSender(s, logical, transport), serverBulkDataSender(s, transport), serverBulkWriteSender(s, logical, transport), serverBulkReleaseSender(s, logical, transport)) bulk.markAcceptHandled() if err := runtime.adoptReserved(scope, bulk); err != nil { runtime.releaseDataID(scope, req.DataID) return nil, err } resp, err := sendBulkOpenServerTransport(ctx, s, transport, req) if err != nil { s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: err.Error()}) bulk.markReset(err) return nil, err } if resp.DataID != 0 && resp.DataID != req.DataID { err = errBulkAlreadyExists s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, Error: "bulk data id mismatch"}) bulk.markReset(err) return nil, err } if resp.FastPathVersion != 0 { bulk.setFastPathVersion(resp.FastPathVersion) } if resp.Dedicated { err = errBulkRejected s.bestEffortBulkResetTransport(transport, BulkResetRequest{BulkID: req.BulkID, DataID: req.DataID, Error: "shared bulk upgraded to dedicated"}) bulk.markReset(err) return nil, err } if resp.AttachToken != "" { bulk.setDedicatedAttachToken(resp.AttachToken) } if resp.TransportGeneration != 0 { bulk.setTransportGeneration(resp.TransportGeneration) } return bulk, nil } func (s *ServerCommon) bestEffortBulkResetLogical(logical *LogicalConn, transport *TransportConn, req BulkResetRequest) { if s == nil || logical == nil { return } task := newServerBulkResetRecoveryTask(s, logical, transport, req) q := s.bulkRecoveryQueue() if !q.enqueue(task) { s.handleBulkRecoveryOverflow(logical, transport, req) } } func (s *ServerCommon) bulkRecoveryQueue() *bulkRecoveryQueue { if s == nil { return nil } s.bulkRecoveryMu.Lock() defer s.bulkRecoveryMu.Unlock() if s.bulkRecovery == nil { s.bulkRecovery = newBulkRecoveryQueue(s.reportBulkRecoveryError) } return s.bulkRecovery } func (s *ServerCommon) bestEffortBulkResetTransport(transport *TransportConn, req BulkResetRequest) { if s == nil || transport == nil { return } task := newServerBulkResetRecoveryTask(s, transport.logicalConnSnapshot(), transport, req) q := s.bulkRecoveryQueue() if !q.enqueue(task) { s.handleBulkRecoveryOverflow(transport.logicalConnSnapshot(), transport, req) } } func newServerBulkResetRecoveryTask(s *ServerCommon, logical *LogicalConn, transport *TransportConn, req BulkResetRequest) bulkRecoveryTask { return func(ctx context.Context) error { if transport == nil { if logical != nil { return transportDetachedErrorForLogical(logical) } return errTransportDetached } _, err := sendBulkResetServerTransport(ctx, s, transport, req) if errors.Is(err, errBulkNotFound) { return nil } return err } } func serverBulkRequest(runtime *bulkRuntime, opt BulkOpenOptions) BulkOpenRequest { opt = normalizeBulkOpenOptions(opt) id := opt.ID if id == "" && runtime != nil { id = runtime.nextID() } return normalizeBulkOpenRequest(BulkOpenRequest{ BulkID: id, FastPathVersion: bulkFastPathVersionCurrent, Range: opt.Range, Metadata: cloneBulkMetadata(opt.Metadata), ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, Dedicated: opt.Dedicated, ChunkSize: opt.ChunkSize, WindowBytes: opt.WindowBytes, MaxInFlight: opt.MaxInFlight, }) } func serverBulkCloseSender(s *ServerCommon, logical *LogicalConn, transport *TransportConn) bulkCloseSender { return func(ctx context.Context, bulk *bulkHandle, full bool) error { if bulk != nil && bulk.Dedicated() { if err := bulk.waitDedicatedReady(ctx); err != nil { return err } return s.sendDedicatedBulkClose(ctx, bulk.LogicalConn(), bulk, full) } req := BulkCloseRequest{ BulkID: bulk.ID(), DataID: bulk.dataIDSnapshot(), Full: full, } if transport != nil { _, err := sendBulkCloseServerTransport(ctx, s, transport, req) return err } _, err := sendBulkCloseServerLogical(ctx, s, logical, req) return err } } func serverBulkResetSender(s *ServerCommon, logical *LogicalConn, transport *TransportConn) bulkResetSender { return func(ctx context.Context, bulk *bulkHandle, message string) error { if bulk != nil && bulk.Dedicated() { if err := bulk.waitDedicatedReady(ctx); err != nil { return err } return s.sendDedicatedBulkReset(ctx, bulk.LogicalConn(), bulk, message) } req := BulkResetRequest{ BulkID: bulk.ID(), DataID: bulk.dataIDSnapshot(), Error: message, } if transport != nil { _, err := sendBulkResetServerTransport(ctx, s, transport, req) return err } _, err := sendBulkResetServerLogical(ctx, s, logical, req) return err } } func serverBulkDataSender(s *ServerCommon, transport *TransportConn) bulkDataSender { return func(ctx context.Context, bulk *bulkHandle, chunk []byte) error { if s == nil { return errBulkServerNil } if ctx != nil { select { case <-ctx.Done(): return ctx.Err() default: } } if bulk != nil && bulk.Dedicated() { if err := bulk.waitDedicatedReady(ctx); err != nil { return err } return s.sendDedicatedBulkData(ctx, bulk.LogicalConn(), bulk, chunk) } if transport == nil { return errBulkTransportNil } if !transport.IsCurrent() { return errTransportDetached } dataID := bulk.dataIDSnapshot() if dataID == 0 { return errBulkDataPathNotReady } return s.sendFastBulkDataTransport(ctx, bulk.LogicalConn(), transport, dataID, bulk.nextOutboundDataSeq(), chunk, bulk.fastPathVersionSnapshot()) } } func serverBulkWriteSender(s *ServerCommon, logical *LogicalConn, transport *TransportConn) bulkWriteSender { return func(ctx context.Context, bulk *bulkHandle, startSeq uint64, payload []byte, payloadOwned bool) (int, error) { if s == nil { return 0, errBulkServerNil } if ctx != nil { select { case <-ctx.Done(): return 0, ctx.Err() default: } } if bulk != nil && bulk.Dedicated() { if err := bulk.waitDedicatedReady(ctx); err != nil { return 0, err } return s.sendDedicatedBulkWrite(ctx, bulk.LogicalConn(), bulk, startSeq, payload, payloadOwned) } if transport == nil { return 0, errBulkTransportNil } if !transport.IsCurrent() { return 0, errTransportDetached } if bulk == nil { return 0, errBulkRuntimeNil } dataID := bulk.dataIDSnapshot() if dataID == 0 { return 0, errBulkDataPathNotReady } return s.sendFastBulkWriteTransport(ctx, bulk.LogicalConn(), transport, dataID, startSeq, bulk.chunkSize, bulk.fastPathVersionSnapshot(), payload, payloadOwned) } } func serverBulkReleaseSender(s *ServerCommon, logical *LogicalConn, transport *TransportConn) bulkReleaseSender { return func(bulk *bulkHandle, bytes int64, chunks int) error { if s == nil || bulk == nil { return errBulkServerNil } if bytes <= 0 && chunks <= 0 { return nil } ctx, cancel, err := bulk.newWriteContext(bulk.Context(), bulk.writeTimeout) if err != nil { return err } defer cancel() if bulk.Dedicated() { return s.sendDedicatedBulkRelease(ctx, logical, bulk, bytes, chunks) } if transport != nil && transport.IsCurrent() && bulk.fastPathVersionSnapshot() >= bulkFastPathVersionV2 { payload, err := encodeBulkDedicatedReleasePayload(bytes, chunks) if err != nil { return err } return s.sendFastBulkControlTransport(ctx, logical, transport, bulkFastPayloadTypeRelease, 0, bulk.dataIDSnapshot(), 0, bulk.fastPathVersionSnapshot(), payload) } req := BulkReleaseRequest{ BulkID: bulk.ID(), DataID: bulk.dataIDSnapshot(), Bytes: bytes, Chunks: chunks, } if transport != nil { return sendBulkReleaseServerTransport(ctx, s, transport, req) } return sendBulkReleaseServerLogical(ctx, s, logical, req) } }