1use std::error::Error;
4use std::fmt;
5use std::time::Duration;
6
7use riffdb_proto::generated::{
8 admin_service_client::AdminServiceClient,
9 application_session_service_client::ApplicationSessionServiceClient,
10 command_service_client::CommandServiceClient, commit_service_client::CommitServiceClient,
11 contract_service_client::ContractServiceClient, event_service_client::EventServiceClient,
12 query_service_client::QueryServiceClient,
13};
14use riffdb_proto::generated_app::application_query_service_client::ApplicationQueryServiceClient;
15use riffdb_proto::{
16 MAX_EXECUTE_REQUEST_BYTES, MAX_EXECUTE_RESPONSE_BYTES, PublicMessage,
17 validate_apply_application_reimport_page_exchange, validate_apply_contract_migration_exchange,
18 validate_cancel_application_export_exchange, validate_cancel_application_reimport_exchange,
19 validate_check_contract_migration_exchange, validate_contract_validation_exchange,
20 validate_create_capability_exchange, validate_create_offline_backup_exchange,
21 validate_discover_command_tools_exchange, validate_discover_resources_exchange,
22 validate_explain_command_exchange, validate_get_application_export_exchange,
23 validate_get_application_export_page_exchange, validate_get_application_installation_exchange,
24 validate_get_application_reimport_exchange, validate_get_contract_migration_operation_exchange,
25 validate_get_contract_version_exchange, validate_get_offline_maintenance_operation_exchange,
26 validate_get_outcome_exchange, validate_get_projection_status_exchange,
27 validate_list_pending_outbox_deliveries_exchange, validate_public_message,
28 validate_query_projection_exchange, validate_restore_archived_backup_exchange, validate_restore_offline_backup_exchange,
29 validate_retire_offline_backup_exchange, validate_scan_commits_exchange,
30 validate_scan_index_exchange, validate_start_application_export_exchange,
31 validate_start_application_installation_exchange, validate_start_application_reimport_exchange,
32 validate_start_offline_storage_scrub_exchange, validate_trace_provenance_exchange,
33};
34use riffdb_proto::{app::v1 as app_v1, v1};
35use riffdb_types::{
36 ApplicationExportOperationId, ApplicationInstallationCampaignId, CommitSequence, RequestId,
37};
38use tonic::transport::{Channel, Endpoint};
39use tonic::{Request, Response, Streaming};
40
41use crate::application::contextualize_command_client_error;
42use crate::command::{RetryDecision, RetryState, apply_overloaded_backoff};
43use crate::generated::{GeneratedCommand, GeneratedCommandError};
44use crate::status::{
45 ClientError, ProtocolFailure, ProtocolFailureKind, checked_application_status, checked_status,
46};
47use crate::{
48 ApplyContractMigration, AttemptBudget, BootstrapCallMetadata,
49 BootstrapCapabilityCreateTemplate, CallMetadata, CheckContractMigration, CreateOfflineBackup,
50 IdempotentCommand, NormalCapabilityCreateTemplate, RestoreArchivedBackup, RestoreOfflineBackup, RetireOfflineBackup,
51 StartApplicationExport, StartApplicationInstallation, StartApplicationReimport,
52 StartOfflineStorageScrub, SystemIdSource,
53};
54
55pub(crate) trait RetryRequestIdSource {
56 fn next_request_id(&mut self) -> Result<RequestId, crate::IdentifierGenerationError>;
57}
58
59impl RetryRequestIdSource for SystemIdSource {
60 fn next_request_id(&mut self) -> Result<RequestId, crate::IdentifierGenerationError> {
61 (*self).request_id()
62 }
63}
64
65pub(crate) trait ExecuteRetryAttempt {
66 async fn submit_execute_attempt(
67 &mut self,
68 request: v1::ExecuteCommandRequest,
69 metadata: &CallMetadata,
70 ) -> Result<v1::ExecuteCommandResponse, ClientError>;
71}
72
73trait NormalCapabilityRetryAttempt {
74 async fn submit_normal_capability_attempt(
75 &mut self,
76 request: v1::CreateCapabilityRequest,
77 metadata: &CallMetadata,
78 ) -> Result<v1::CreateCapabilityResponse, ClientError>;
79}
80
81trait BootstrapCapabilityRetryAttempt {
82 async fn submit_bootstrap_capability_attempt(
83 &mut self,
84 request: v1::CreateCapabilityRequest,
85 metadata: &BootstrapCallMetadata,
86 ) -> Result<v1::CreateCapabilityResponse, ClientError>;
87}
88
89macro_rules! unary {
90 ($name:ident, $client:ident, $rpc:ident, $request:ty, $response:ty) => {
91 #[doc = concat!("Performs the checked `", stringify!($rpc), "` unary RPC.")]
92 pub async fn $name(
93 &mut self,
94 message: $request,
95 metadata: &CallMetadata,
96 ) -> Result<$response, ClientError> {
97 validate_outbound(&message)?;
105 let mut request = Request::new(message);
106 metadata.apply(&mut request);
107 let response = self
108 .$client
109 .$rpc(request)
110 .await
111 .map_err(checked_status)?
112 .into_inner();
113 Ok(response)
114 }
115 };
116}
117
118macro_rules! unary_exchange {
119 ($name:ident, $client:ident, $rpc:ident, $request:ty, $response:ty, $exchange:path) => {
120 #[doc = concat!("Performs the checked `", stringify!($rpc), "` unary RPC.")]
121 pub async fn $name(
122 &mut self,
123 message: $request,
124 metadata: &CallMetadata,
125 ) -> Result<$response, ClientError> {
126 validate_outbound(&message)?;
127 let exchange_request = message.clone();
128 let mut request = Request::new(message);
129 metadata.apply(&mut request);
130 let response = self
131 .$client
132 .$rpc(request)
133 .await
134 .map_err(checked_status)?
135 .into_inner();
136 $exchange(&exchange_request, &response).map_err(|_| invalid_inbound())?;
141 Ok(response)
142 }
143 };
144}
145
146macro_rules! unary_application {
147 ($name:ident, $rpc:ident, $request:ty, $response:ty) => {
148 #[doc = concat!(
149 "Performs the checked application `",
150 stringify!($rpc),
151 "` unary RPC."
152 )]
153 pub async fn $name(
154 &mut self,
155 message: $request,
156 metadata: &CallMetadata,
157 ) -> Result<$response, ClientError> {
158 validate_outbound(&message)?;
159 let mut request = Request::new(message);
160 metadata.apply(&mut request);
161 let response = self
162 .application_query
163 .$rpc(request)
164 .await
165 .map_err(checked_application_status)?
166 .into_inner();
167 Ok(response)
168 }
169 };
170}
171
172#[must_use]
174pub(crate) const fn merge_observed_history_incarnation(
175 explicit: Option<u64>,
176 remembered: Option<u64>,
177) -> Option<u64> {
178 if explicit.is_some() {
179 explicit
180 } else {
181 remembered
182 }
183}
184
185#[derive(Clone)]
190pub struct RiffDbClient {
191 contract: ContractServiceClient<Channel>,
192 command: CommandServiceClient<Channel>,
193 query: QueryServiceClient<Channel>,
194 commit: CommitServiceClient<Channel>,
195 event: EventServiceClient<Channel>,
196 admin: AdminServiceClient<Channel>,
197 application_query: ApplicationQueryServiceClient<Channel>,
198 application_session: ApplicationSessionServiceClient<Channel>,
199 bounded_application_session: Option<crate::session::BoundedApplicationSession>,
200 observed_history_incarnation: Option<u64>,
206}
207
208impl RiffDbClient {
209 pub async fn connect(endpoint: Endpoint) -> Result<Self, ClientError> {
211 let channel = endpoint
212 .connect()
213 .await
214 .map_err(|_| ClientError::ConnectionFailure)?;
215 Ok(Self::from_channel(channel))
216 }
217
218 #[must_use]
220 pub fn from_channel(channel: Channel) -> Self {
221 Self {
222 contract: ContractServiceClient::new(channel.clone()),
223 command: CommandServiceClient::new(channel.clone())
224 .max_encoding_message_size(MAX_EXECUTE_REQUEST_BYTES)
225 .max_decoding_message_size(MAX_EXECUTE_RESPONSE_BYTES),
226 query: QueryServiceClient::new(channel.clone()),
227 commit: CommitServiceClient::new(channel.clone()),
228 event: EventServiceClient::new(channel.clone())
229 .max_encoding_message_size(MAX_EXECUTE_REQUEST_BYTES)
230 .max_decoding_message_size(MAX_EXECUTE_RESPONSE_BYTES),
231 admin: AdminServiceClient::new(channel.clone()),
232 application_query: ApplicationQueryServiceClient::new(channel.clone()),
233 application_session: ApplicationSessionServiceClient::new(channel)
234 .max_encoding_message_size(MAX_EXECUTE_REQUEST_BYTES)
235 .max_decoding_message_size(MAX_EXECUTE_RESPONSE_BYTES),
236 bounded_application_session: None,
237 observed_history_incarnation: None,
238 }
239 }
240
241 pub(crate) async fn enable_bounded_application_session(
242 &mut self,
243 identity: crate::ApplicationSessionIdentity,
244 metadata: CallMetadata,
245 ) -> Result<(), ClientError> {
246 let session =
247 crate::session::BoundedApplicationSession::open(self, identity, metadata).await?;
248 self.bounded_application_session = Some(session);
249 Ok(())
250 }
251
252 pub(crate) const fn bounded_application_session_enabled(&self) -> bool {
253 self.bounded_application_session.is_some()
254 }
255
256 pub(crate) fn disable_bounded_application_session(&mut self) {
257 if let Some(session) = self.bounded_application_session.take() {
258 session.close();
259 }
260 }
261
262 pub(crate) async fn execute_selected_query_transport(
263 &mut self,
264 message: app_v1::ExecuteQueryRequest,
265 metadata: &CallMetadata,
266 ) -> Result<app_v1::ExecuteQueryResponse, ClientError> {
267 if let Some(session) = self.bounded_application_session.as_ref() {
268 if !session.matches_metadata(metadata) {
269 return Err(invalid_outbound());
270 }
271 return session.execute_query(message).await;
272 }
273 self.execute_query(message, metadata).await
274 }
275
276 pub(crate) async fn open_application_session<S>(
277 &mut self,
278 stream: S,
279 metadata: &CallMetadata,
280 ) -> Result<Streaming<v1::ApplicationSessionResponse>, ClientError>
281 where
282 S: futures_util::Stream<Item = v1::ApplicationSessionRequest> + Send + 'static,
283 {
284 let mut request = Request::new(stream);
285 metadata.apply(&mut request);
286 self.application_session
287 .open(request)
288 .await
289 .map_err(checked_application_status)
290 .map(Response::into_inner)
291 }
292
293 pub fn set_observed_history_incarnation(&mut self, observed: Option<u64>) {
297 self.observed_history_incarnation = observed;
298 }
299
300 #[must_use]
302 pub const fn observed_history_incarnation(&self) -> Option<u64> {
303 self.observed_history_incarnation
304 }
305
306 #[must_use]
308 pub fn apply_observed_history_incarnation_get(
309 &self,
310 mut request: v1::GetCommitRequest,
311 ) -> v1::GetCommitRequest {
312 request.observed_history_incarnation = merge_observed_history_incarnation(
313 request.observed_history_incarnation,
314 self.observed_history_incarnation,
315 );
316 request
317 }
318
319 #[must_use]
321 pub fn apply_observed_history_incarnation_scan(
322 &self,
323 mut request: v1::ScanCommitsRequest,
324 ) -> v1::ScanCommitsRequest {
325 request.observed_history_incarnation = merge_observed_history_incarnation(
326 request.observed_history_incarnation,
327 self.observed_history_incarnation,
328 );
329 request
330 }
331
332 #[must_use]
334 pub fn apply_observed_history_incarnation_subscribe(
335 &self,
336 mut request: v1::SubscribeCommitsRequest,
337 ) -> v1::SubscribeCommitsRequest {
338 request.observed_history_incarnation = merge_observed_history_incarnation(
339 request.observed_history_incarnation,
340 self.observed_history_incarnation,
341 );
342 request
343 }
344
345 #[must_use]
347 pub fn apply_observed_history_incarnation_replay(
348 &self,
349 mut request: v1::ReplayEventsRequest,
350 ) -> v1::ReplayEventsRequest {
351 if request.observed_history_incarnation == 0
352 && let Some(remembered) = self.observed_history_incarnation
353 {
354 request.observed_history_incarnation = remembered;
355 }
356 request
357 }
358
359 #[must_use]
361 pub fn apply_observed_history_incarnation_tail(
362 &self,
363 mut request: v1::TailEventsRequest,
364 ) -> v1::TailEventsRequest {
365 if request.observed_history_incarnation == 0
366 && let Some(remembered) = self.observed_history_incarnation
367 {
368 request.observed_history_incarnation = remembered;
369 }
370 request
371 }
372
373 pub fn remember_history_incarnation_from_get(&mut self, response: &v1::GetCommitResponse) {
375 if response.history_incarnation >= 1 {
376 self.observed_history_incarnation = Some(response.history_incarnation);
377 }
378 }
379
380 pub fn remember_history_incarnation_from_scan(&mut self, response: &v1::ScanCommitsResponse) {
382 if let Some(page) = response.page.as_ref()
383 && page.history_incarnation >= 1
384 {
385 self.observed_history_incarnation = Some(page.history_incarnation);
386 }
387 }
388
389 pub fn remember_history_incarnation_from_event_page(&mut self, page: Option<&v1::EventPage>) {
391 if let Some(page) = page
392 && page.history_incarnation >= 1
393 {
394 self.observed_history_incarnation = Some(page.history_incarnation);
395 }
396 }
397
398 unary_application!(
399 describe_contract,
400 describe_contract,
401 app_v1::DescribeContractRequest,
402 app_v1::DescribeContractResponse
403 );
404 unary_application!(
405 get_application_catalog,
406 get_application_catalog,
407 app_v1::GetApplicationCatalogRequest,
408 app_v1::GetApplicationCatalogResponse
409 );
410 unary_application!(
411 check_query,
412 check_query,
413 app_v1::CheckQueryRequest,
414 app_v1::CheckQueryResponse
415 );
416 unary_application!(
417 explain_query,
418 explain_query,
419 app_v1::ExplainQueryRequest,
420 app_v1::ExplainQueryResponse
421 );
422 unary_application!(
423 execute_query,
424 execute_query,
425 app_v1::ExecuteQueryRequest,
426 app_v1::ExecuteQueryResponse
427 );
428 unary_application!(
429 deploy_query_module,
430 deploy_query_module,
431 app_v1::DeployQueryModuleRequest,
432 app_v1::DeployQueryModuleResponse
433 );
434 unary_application!(
435 deploy_reactive_module,
436 deploy_reactive_module,
437 app_v1::DeployReactiveModuleRequest,
438 app_v1::DeployReactiveModuleResponse
439 );
440 unary_application!(
441 get_query_module,
442 get_query_module,
443 app_v1::GetQueryModuleRequest,
444 app_v1::GetQueryModuleResponse
445 );
446 unary_application!(
447 execute_projected_query_raw,
448 execute_projected_query,
449 app_v1::ExecuteProjectedQueryRequest,
450 app_v1::ExecuteProjectedQueryResponse
451 );
452 unary_application!(
453 inspect_vector_state,
454 inspect_vector_state,
455 app_v1::InspectVectorStateRequest,
456 app_v1::InspectVectorStateResponse
457 );
458
459 unary_exchange!(
460 validate_contract,
461 contract,
462 validate_contract,
463 v1::ValidateContractRequest,
464 v1::ValidateContractResponse,
465 validate_contract_validation_exchange
466 );
467 unary_exchange!(
468 explain_command,
469 contract,
470 explain_command,
471 v1::ExplainCommandRequest,
472 v1::ExplainCommandResponse,
473 validate_explain_command_exchange
474 );
475 unary!(
476 deploy_contract,
477 contract,
478 deploy_contract,
479 v1::DeployContractRequest,
480 v1::DeployContractResponse
481 );
482 unary!(
483 get_active_contract,
484 contract,
485 get_active_contract,
486 v1::GetActiveContractRequest,
487 v1::GetActiveContractResponse
488 );
489 unary_exchange!(
490 get_contract_version,
491 contract,
492 get_contract_version,
493 v1::GetContractVersionRequest,
494 v1::GetContractVersionResponse,
495 validate_get_contract_version_exchange
496 );
497 unary_exchange!(
498 discover_command_tools,
499 contract,
500 discover_command_tools,
501 v1::DiscoverCommandToolsRequest,
502 v1::DiscoverCommandToolsResponse,
503 validate_discover_command_tools_exchange
504 );
505 unary_exchange!(
506 discover_resources,
507 contract,
508 discover_resources,
509 v1::DiscoverResourcesRequest,
510 v1::DiscoverResourcesResponse,
511 validate_discover_resources_exchange
512 );
513 unary!(
514 get_reactive_wakeup,
515 contract,
516 get_reactive_wakeup,
517 v1::GetReactiveWakeupRequest,
518 v1::GetReactiveWakeupResponse
519 );
520 unary!(
521 execute,
522 command,
523 execute,
524 v1::ExecuteCommandRequest,
525 v1::ExecuteCommandResponse
526 );
527 unary_exchange!(
528 execute_batch,
529 command,
530 execute_batch,
531 v1::ExecuteCommandBatchRequest,
532 v1::ExecuteCommandBatchResponse,
533 riffdb_proto::validate_execute_command_batch_exchange
534 );
535 unary_exchange!(
536 get_outcome,
537 command,
538 get_outcome,
539 v1::GetOutcomeRequest,
540 v1::GetOutcomeResponse,
541 validate_get_outcome_exchange
542 );
543 unary!(
544 get_entity,
545 query,
546 get_entity,
547 v1::GetEntityRequest,
548 v1::GetEntityResponse
549 );
550 unary_exchange!(
551 scan_index,
552 query,
553 scan_index,
554 v1::ScanIndexRequest,
555 v1::ScanIndexResponse,
556 validate_scan_index_exchange
557 );
558 unary_exchange!(
559 query_projection,
560 query,
561 query_projection,
562 v1::QueryProjectionRequest,
563 v1::QueryProjectionResponse,
564 validate_query_projection_exchange
565 );
566 unary_exchange!(
567 get_projection_status,
568 query,
569 get_projection_status,
570 v1::GetProjectionStatusRequest,
571 v1::GetProjectionStatusResponse,
572 validate_get_projection_status_exchange
573 );
574
575 pub async fn wait_for_projection(
581 &mut self,
582 mut message: v1::QueryProjectionRequest,
583 required_sequence: CommitSequence,
584 maximum_wait: Duration,
585 metadata: &CallMetadata,
586 ) -> Result<v1::QueryProjectionResponse, ClientError> {
587 configure_projection_wait(&mut message, required_sequence, maximum_wait)?;
588 self.query_projection(message, metadata).await
589 }
590 pub async fn get_commit(
592 &mut self,
593 message: v1::GetCommitRequest,
594 metadata: &CallMetadata,
595 ) -> Result<v1::GetCommitResponse, ClientError> {
596 let message = self.apply_observed_history_incarnation_get(message);
597 validate_outbound(&message)?;
598 let mut request = Request::new(message);
599 metadata.apply(&mut request);
600 let response = self
601 .commit
602 .get_commit(request)
603 .await
604 .map_err(checked_status)?
605 .into_inner();
606 self.remember_history_incarnation_from_get(&response);
607 Ok(response)
608 }
609
610 pub async fn scan_commits(
612 &mut self,
613 message: v1::ScanCommitsRequest,
614 metadata: &CallMetadata,
615 ) -> Result<v1::ScanCommitsResponse, ClientError> {
616 let message = self.apply_observed_history_incarnation_scan(message);
617 validate_outbound(&message)?;
618 let exchange_request = message.clone();
619 let mut request = Request::new(message);
620 metadata.apply(&mut request);
621 let response = self
622 .commit
623 .scan_commits(request)
624 .await
625 .map_err(checked_status)?
626 .into_inner();
627 validate_scan_commits_exchange(&exchange_request, &response)
628 .map_err(|_| invalid_inbound())?;
629 self.remember_history_incarnation_from_scan(&response);
630 Ok(response)
631 }
632
633 unary_exchange!(
634 trace_provenance,
635 commit,
636 trace_provenance,
637 v1::TraceProvenanceRequest,
638 v1::TraceProvenanceResponse,
639 validate_trace_provenance_exchange
640 );
641 unary!(
642 describe_event,
643 event,
644 describe_event,
645 v1::DescribeEventRequest,
646 v1::DescribeEventResponse
647 );
648
649 pub async fn replay_events(
651 &mut self,
652 message: v1::ReplayEventsRequest,
653 metadata: &CallMetadata,
654 ) -> Result<v1::ReplayEventsResponse, ClientError> {
655 let message = self.apply_observed_history_incarnation_replay(message);
656 validate_outbound(&message)?;
657 let mut request = Request::new(message);
658 metadata.apply(&mut request);
659 let response = self
660 .event
661 .replay_events(request)
662 .await
663 .map_err(checked_status)?
664 .into_inner();
665 self.remember_history_incarnation_from_event_page(response.page.as_ref());
666 Ok(response)
667 }
668
669 pub async fn tail_events(
671 &mut self,
672 message: v1::TailEventsRequest,
673 metadata: &CallMetadata,
674 ) -> Result<v1::TailEventsResponse, ClientError> {
675 let message = self.apply_observed_history_incarnation_tail(message);
676 validate_outbound(&message)?;
677 let mut request = Request::new(message);
678 metadata.apply(&mut request);
679 let response = self
680 .event
681 .tail_events(request)
682 .await
683 .map_err(checked_status)?
684 .into_inner();
685 self.remember_history_incarnation_from_event_page(response.page.as_ref());
686 Ok(response)
687 }
688 unary!(
689 consume_event_stream,
690 event,
691 consume_event_stream,
692 v1::ConsumeEventStreamRequest,
693 v1::ConsumeEventStreamResponse
694 );
695 unary!(
696 acknowledge_event_stream,
697 event,
698 acknowledge_event_stream,
699 v1::AcknowledgeEventStreamRequest,
700 v1::EventConsumerMutationResponse
701 );
702 unary!(
703 negative_acknowledge_event_stream,
704 event,
705 negative_acknowledge_event_stream,
706 v1::NegativeAcknowledgeEventStreamRequest,
707 v1::EventConsumerMutationResponse
708 );
709 unary!(
710 seek_event_stream_consumer,
711 event,
712 seek_event_stream_consumer,
713 v1::SeekEventStreamConsumerRequest,
714 v1::EventConsumerMutationResponse
715 );
716 unary!(
717 retire_event_stream_consumer,
718 event,
719 retire_event_stream_consumer,
720 v1::RetireEventStreamConsumerRequest,
721 v1::EventConsumerMutationResponse
722 );
723 unary!(
724 get_event_stream_consumer_status,
725 event,
726 get_event_stream_consumer_status,
727 v1::GetEventStreamConsumerStatusRequest,
728 v1::GetEventStreamConsumerStatusResponse
729 );
730 unary!(
731 consume_contextual_subscription,
732 event,
733 consume_contextual_subscription,
734 v1::ConsumeContextualSubscriptionRequest,
735 v1::ConsumeContextualSubscriptionResponse
736 );
737 unary!(
738 acknowledge_contextual_subscription,
739 event,
740 acknowledge_contextual_subscription,
741 v1::AcknowledgeContextualSubscriptionRequest,
742 v1::EventConsumerMutationResponse
743 );
744 unary!(
745 negative_acknowledge_contextual_subscription,
746 event,
747 negative_acknowledge_contextual_subscription,
748 v1::NegativeAcknowledgeContextualSubscriptionRequest,
749 v1::EventConsumerMutationResponse
750 );
751 unary!(
752 get_contextual_subscription_status,
753 event,
754 get_contextual_subscription_status,
755 v1::GetContextualSubscriptionStatusRequest,
756 v1::GetEventStreamConsumerStatusResponse
757 );
758 unary!(
759 execute_contextual_reaction,
760 event,
761 execute_contextual_reaction,
762 v1::ExecuteContextualReactionRequest,
763 v1::ExecuteCommandResponse
764 );
765 unary!(register_follower, admin, register_follower, v1::RegisterFollowerRequest, v1::RegisterFollowerResponse);
766 unary_exchange!(promote_follower, admin, promote_follower, v1::PromoteFollowerRequest, v1::PromoteFollowerResponse, riffdb_proto::validate_promote_follower_exchange);
767 unary_exchange!(retire_follower, admin, retire_follower, v1::RetireFollowerRequest, v1::RetireFollowerResponse, riffdb_proto::validate_retire_follower_exchange);
768 unary_exchange!(fence_replication_primary, admin, fence_replication_primary, v1::FenceReplicationPrimaryRequest, v1::FenceReplicationPrimaryResponse, riffdb_proto::validate_fence_replication_primary_exchange);
769 unary!(health, admin, health, v1::HealthRequest, v1::HealthResponse);
770 unary!(stats, admin, stats, v1::StatsRequest, v1::StatsResponse);
771 unary!(
772 revoke_capability,
773 admin,
774 revoke_capability,
775 v1::RevokeCapabilityRequest,
776 v1::RevokeCapabilityResponse
777 );
778 unary_exchange!(
779 list_pending_outbox_deliveries,
780 admin,
781 list_pending_outbox_deliveries,
782 v1::ListPendingOutboxDeliveriesRequest,
783 v1::ListPendingOutboxDeliveriesResponse,
784 validate_list_pending_outbox_deliveries_exchange
785 );
786 unary_exchange!(
787 create_offline_backup,
788 admin,
789 create_offline_backup,
790 v1::CreateOfflineBackupRequest,
791 v1::CreateOfflineBackupResponse,
792 validate_create_offline_backup_exchange
793 );
794 unary_exchange!(
795 restore_offline_backup,
796 admin,
797 restore_offline_backup,
798 v1::RestoreOfflineBackupRequest,
799 v1::RestoreOfflineBackupResponse,
800 validate_restore_offline_backup_exchange
801 );
802 unary_exchange!(
803 restore_archived_backup,
804 admin,
805 restore_archived_backup,
806 v1::RestoreArchivedBackupRequest,
807 v1::RestoreArchivedBackupResponse,
808 validate_restore_archived_backup_exchange
809 );
810 unary_exchange!(
811 retire_offline_backup,
812 admin,
813 retire_offline_backup,
814 v1::RetireOfflineBackupRequest,
815 v1::RetireOfflineBackupResponse,
816 validate_retire_offline_backup_exchange
817 );
818 unary_exchange!(
819 start_offline_storage_scrub,
820 admin,
821 start_offline_storage_scrub,
822 v1::StartOfflineStorageScrubRequest,
823 v1::StartOfflineStorageScrubResponse,
824 validate_start_offline_storage_scrub_exchange
825 );
826 unary_exchange!(
827 get_offline_maintenance_operation,
828 admin,
829 get_offline_maintenance_operation,
830 v1::GetOfflineMaintenanceOperationRequest,
831 v1::GetOfflineMaintenanceOperationResponse,
832 validate_get_offline_maintenance_operation_exchange
833 );
834 unary_exchange!(
835 check_contract_migration,
836 admin,
837 check_contract_migration,
838 v1::CheckContractMigrationRequest,
839 v1::CheckContractMigrationResponse,
840 validate_check_contract_migration_exchange
841 );
842 unary_exchange!(
843 apply_contract_migration,
844 admin,
845 apply_contract_migration,
846 v1::ApplyContractMigrationRequest,
847 v1::ApplyContractMigrationResponse,
848 validate_apply_contract_migration_exchange
849 );
850 unary_exchange!(
851 get_contract_migration_operation,
852 admin,
853 get_contract_migration_operation,
854 v1::GetContractMigrationOperationRequest,
855 v1::GetContractMigrationOperationResponse,
856 validate_get_contract_migration_operation_exchange
857 );
858 unary_exchange!(
859 start_application_installation,
860 admin,
861 start_application_installation,
862 v1::StartApplicationInstallationRequest,
863 v1::StartApplicationInstallationResponse,
864 validate_start_application_installation_exchange
865 );
866 unary_exchange!(
867 get_application_installation,
868 admin,
869 get_application_installation,
870 v1::GetApplicationInstallationRequest,
871 v1::GetApplicationInstallationResponse,
872 validate_get_application_installation_exchange
873 );
874 unary_exchange!(
875 start_application_export,
876 admin,
877 start_application_export,
878 v1::StartApplicationExportRequest,
879 v1::StartApplicationExportResponse,
880 validate_start_application_export_exchange
881 );
882 unary_exchange!(
883 get_application_export_page,
884 admin,
885 get_application_export_page,
886 v1::GetApplicationExportPageRequest,
887 v1::GetApplicationExportPageResponse,
888 validate_get_application_export_page_exchange
889 );
890 unary_exchange!(
891 get_application_export,
892 admin,
893 get_application_export,
894 v1::GetApplicationExportRequest,
895 v1::GetApplicationExportResponse,
896 validate_get_application_export_exchange
897 );
898 unary_exchange!(
899 cancel_application_export,
900 admin,
901 cancel_application_export,
902 v1::CancelApplicationExportRequest,
903 v1::CancelApplicationExportResponse,
904 validate_cancel_application_export_exchange
905 );
906 unary_exchange!(
907 start_application_reimport,
908 admin,
909 start_application_reimport,
910 v1::StartApplicationReimportRequest,
911 v1::StartApplicationReimportResponse,
912 validate_start_application_reimport_exchange
913 );
914 unary_exchange!(
915 apply_application_reimport_page,
916 admin,
917 apply_application_reimport_page,
918 v1::ApplyApplicationReimportPageRequest,
919 v1::ApplyApplicationReimportPageResponse,
920 validate_apply_application_reimport_page_exchange
921 );
922 unary_exchange!(
923 get_application_reimport,
924 admin,
925 get_application_reimport,
926 v1::GetApplicationReimportRequest,
927 v1::GetApplicationReimportResponse,
928 validate_get_application_reimport_exchange
929 );
930 unary_exchange!(
931 cancel_application_reimport,
932 admin,
933 cancel_application_reimport,
934 v1::CancelApplicationReimportRequest,
935 v1::CancelApplicationReimportResponse,
936 validate_cancel_application_reimport_exchange
937 );
938
939 pub async fn create_capability(
941 &mut self,
942 message: v1::CreateCapabilityRequest,
943 metadata: &CallMetadata,
944 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
945 if message.mode != v1::CapabilityCreateMode::Normal as i32 {
946 return Err(invalid_outbound());
947 }
948 self.create_capability_with(message, metadata, CallMetadata::apply)
949 .await
950 }
951
952 pub async fn create_bootstrap_capability(
954 &mut self,
955 message: v1::CreateCapabilityRequest,
956 metadata: &BootstrapCallMetadata,
957 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
958 if message.mode != v1::CapabilityCreateMode::Bootstrap as i32 {
959 return Err(invalid_outbound());
960 }
961 self.create_capability_with(message, metadata, BootstrapCallMetadata::apply)
962 .await
963 }
964
965 async fn create_capability_with<M>(
966 &mut self,
967 message: v1::CreateCapabilityRequest,
968 metadata: &M,
969 apply: fn(&M, &mut Request<v1::CreateCapabilityRequest>),
970 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
971 validate_outbound(&message)?;
972 let exchange_request = message.clone();
973 let mut request = Request::new(message);
974 apply(metadata, &mut request);
975 let response = self
976 .admin
977 .create_capability(request)
978 .await
979 .map_err(checked_status)?
980 .into_inner();
981 validate_create_capability_exchange(&exchange_request, &response)
982 .map_err(|_| invalid_inbound())?;
983 Ok(response)
984 }
985
986 pub async fn subscribe_commits(
992 &mut self,
993 message: v1::SubscribeCommitsRequest,
994 metadata: &CallMetadata,
995 ) -> Result<CommitNotificationStream, ClientError> {
996 let message = self.apply_observed_history_incarnation_subscribe(message);
997 validate_outbound(&message)?;
998 let mut request = Request::new(message);
999 metadata.apply(&mut request);
1000 let inner = self
1001 .commit
1002 .subscribe_commits(request)
1003 .await
1004 .map_err(checked_status)?
1005 .into_inner();
1006 Ok(CommitNotificationStream { inner })
1007 }
1008
1009 pub async fn stream_event_consumer(
1014 &mut self,
1015 message: v1::ConsumeEventStreamRequest,
1016 metadata: &CallMetadata,
1017 ) -> Result<EventConsumerResponseStream, ClientError> {
1018 validate_outbound(&message)?;
1019 let mut request = Request::new(message);
1020 metadata.apply(&mut request);
1021 let inner = self
1022 .event
1023 .stream_event_consumer(request)
1024 .await
1025 .map_err(checked_status)?
1026 .into_inner();
1027 Ok(EventConsumerResponseStream { inner })
1028 }
1029
1030 pub async fn watch_named_query(
1035 &mut self,
1036 message: v1::WatchNamedQueryRequest,
1037 metadata: &CallMetadata,
1038 ) -> Result<LiveQueryUpdateStream, ClientError> {
1039 validate_outbound(&message)?;
1040 let mut request = Request::new(message);
1041 metadata.apply(&mut request);
1042 let inner = self
1043 .query
1044 .watch_named_query(request)
1045 .await
1046 .map_err(checked_status)?
1047 .into_inner();
1048 Ok(LiveQueryUpdateStream { inner })
1049 }
1050
1051 pub async fn execute_with_retry(
1059 &mut self,
1060 command: &IdempotentCommand,
1061 attempt_budget: AttemptBudget,
1062 metadata: &CallMetadata,
1063 ) -> Result<v1::ExecuteCommandResponse, ClientError> {
1064 let mut request_ids = SystemIdSource::new();
1065 execute_retry_attempts(self, command, attempt_budget, metadata, &mut request_ids).await
1066 }
1067
1068 pub async fn create_capability_with_retry(
1070 &mut self,
1071 create: &NormalCapabilityCreateTemplate,
1072 attempt_budget: AttemptBudget,
1073 metadata: &CallMetadata,
1074 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
1075 let mut request_ids = SystemIdSource::new();
1076 normal_capability_retry_attempts(self, create, attempt_budget, metadata, &mut request_ids)
1077 .await
1078 }
1079
1080 pub async fn create_bootstrap_capability_with_retry(
1082 &mut self,
1083 create: &BootstrapCapabilityCreateTemplate,
1084 attempt_budget: AttemptBudget,
1085 metadata: &BootstrapCallMetadata,
1086 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
1087 let mut request_ids = SystemIdSource::new();
1088 bootstrap_capability_retry_attempts(
1089 self,
1090 create,
1091 attempt_budget,
1092 metadata,
1093 &mut request_ids,
1094 )
1095 .await
1096 }
1097
1098 pub async fn create_offline_backup_with_retry(
1100 &mut self,
1101 create: &CreateOfflineBackup,
1102 attempt_budget: AttemptBudget,
1103 metadata: &CallMetadata,
1104 ) -> Result<v1::CreateOfflineBackupResponse, ClientError> {
1105 let mut request_ids = SystemIdSource::new();
1106 let mut retry = RetryState::new(attempt_budget);
1107 loop {
1108 if !retry.begin_submission() {
1109 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1110 }
1111 let request_id = request_ids
1112 .next_request_id()
1113 .map_err(|error| retry.request_id_failure(error))?;
1114 retry.note_request_id(&request_id);
1115 match self
1116 .create_offline_backup(create.request(request_id), metadata)
1117 .await
1118 {
1119 Ok(response) => return Ok(response),
1120 Err(error) => match retry.handle_failure(error) {
1121 RetryDecision::Retry => {}
1122 RetryDecision::RetryAfter(delay) => {
1123 apply_overloaded_backoff(delay).await;
1124 }
1125 RetryDecision::Return(error) => return Err(error),
1126 },
1127 }
1128 }
1129 }
1130
1131 pub async fn restore_offline_backup_with_retry(
1133 &mut self,
1134 restore: &RestoreOfflineBackup,
1135 attempt_budget: AttemptBudget,
1136 metadata: &CallMetadata,
1137 ) -> Result<v1::RestoreOfflineBackupResponse, ClientError> {
1138 let mut request_ids = SystemIdSource::new();
1139 let mut retry = RetryState::new(attempt_budget);
1140 loop {
1141 if !retry.begin_submission() {
1142 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1143 }
1144 let request_id = request_ids
1145 .next_request_id()
1146 .map_err(|error| retry.request_id_failure(error))?;
1147 retry.note_request_id(&request_id);
1148 match self
1149 .restore_offline_backup(restore.request(request_id), metadata)
1150 .await
1151 {
1152 Ok(response) => return Ok(response),
1153 Err(error) => match retry.handle_failure(error) {
1154 RetryDecision::Retry => {}
1155 RetryDecision::RetryAfter(delay) => {
1156 apply_overloaded_backoff(delay).await;
1157 }
1158 RetryDecision::Return(error) => return Err(error),
1159 },
1160 }
1161 }
1162 }
1163
1164 pub async fn restore_archived_backup_with_retry(
1166 &mut self,
1167 restore: &RestoreArchivedBackup,
1168 attempt_budget: AttemptBudget,
1169 metadata: &CallMetadata,
1170 ) -> Result<v1::RestoreArchivedBackupResponse, ClientError> {
1171 let mut request_ids = SystemIdSource::new();
1172 let mut retry = RetryState::new(attempt_budget);
1173 loop {
1174 if !retry.begin_submission() {
1175 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1176 }
1177 let request_id = request_ids
1178 .next_request_id()
1179 .map_err(|error| retry.request_id_failure(error))?;
1180 retry.note_request_id(&request_id);
1181 match self
1182 .restore_archived_backup(restore.request(request_id), metadata)
1183 .await
1184 {
1185 Ok(response) => return Ok(response),
1186 Err(error) => match retry.handle_failure(error) {
1187 RetryDecision::Retry => {}
1188 RetryDecision::RetryAfter(delay) => {
1189 apply_overloaded_backoff(delay).await;
1190 }
1191 RetryDecision::Return(error) => return Err(error),
1192 },
1193 }
1194 }
1195 }
1196
1197 pub async fn retire_offline_backup_with_retry(
1199 &mut self,
1200 retire: &RetireOfflineBackup,
1201 attempt_budget: AttemptBudget,
1202 metadata: &CallMetadata,
1203 ) -> Result<v1::RetireOfflineBackupResponse, ClientError> {
1204 let mut request_ids = SystemIdSource::new();
1205 let mut retry = RetryState::new(attempt_budget);
1206 loop {
1207 if !retry.begin_submission() {
1208 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1209 }
1210 let request_id = request_ids
1211 .next_request_id()
1212 .map_err(|error| retry.request_id_failure(error))?;
1213 retry.note_request_id(&request_id);
1214 match self
1215 .retire_offline_backup(retire.request(request_id), metadata)
1216 .await
1217 {
1218 Ok(response) => return Ok(response),
1219 Err(error) => match retry.handle_failure(error) {
1220 RetryDecision::Retry => {}
1221 RetryDecision::RetryAfter(delay) => {
1222 apply_overloaded_backoff(delay).await;
1223 }
1224 RetryDecision::Return(error) => return Err(error),
1225 },
1226 }
1227 }
1228 }
1229
1230 pub async fn start_offline_storage_scrub_with_retry(
1232 &mut self,
1233 scrub: &StartOfflineStorageScrub,
1234 attempt_budget: AttemptBudget,
1235 metadata: &CallMetadata,
1236 ) -> Result<v1::StartOfflineStorageScrubResponse, ClientError> {
1237 let mut request_ids = SystemIdSource::new();
1238 let mut retry = RetryState::new(attempt_budget);
1239 loop {
1240 if !retry.begin_submission() {
1241 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1242 }
1243 let request_id = request_ids
1244 .next_request_id()
1245 .map_err(|error| retry.request_id_failure(error))?;
1246 retry.note_request_id(&request_id);
1247 match self
1248 .start_offline_storage_scrub(scrub.request(request_id), metadata)
1249 .await
1250 {
1251 Ok(response) => return Ok(response),
1252 Err(error) => match retry.handle_failure(error) {
1253 RetryDecision::Retry => {}
1254 RetryDecision::RetryAfter(delay) => {
1255 apply_overloaded_backoff(delay).await;
1256 }
1257 RetryDecision::Return(error) => return Err(error),
1258 },
1259 }
1260 }
1261 }
1262
1263 pub async fn check_contract_migration_with_retry(
1265 &mut self,
1266 check: &CheckContractMigration,
1267 attempt_budget: AttemptBudget,
1268 metadata: &CallMetadata,
1269 ) -> Result<v1::CheckContractMigrationResponse, ClientError> {
1270 let mut request_ids = SystemIdSource::new();
1271 let mut retry = RetryState::new(attempt_budget);
1272 loop {
1273 if !retry.begin_submission() {
1274 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1275 }
1276 let request_id = request_ids
1277 .next_request_id()
1278 .map_err(|error| retry.request_id_failure(error))?;
1279 retry.note_request_id(&request_id);
1280 match self
1281 .check_contract_migration(check.request(request_id), metadata)
1282 .await
1283 {
1284 Ok(response) => return Ok(response),
1285 Err(error) => match retry.handle_failure(error) {
1286 RetryDecision::Retry => {}
1287 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1288 RetryDecision::Return(error) => return Err(error),
1289 },
1290 }
1291 }
1292 }
1293
1294 pub async fn apply_contract_migration_with_retry(
1296 &mut self,
1297 apply: &ApplyContractMigration,
1298 attempt_budget: AttemptBudget,
1299 metadata: &CallMetadata,
1300 ) -> Result<v1::ApplyContractMigrationResponse, ClientError> {
1301 let mut request_ids = SystemIdSource::new();
1302 let mut retry = RetryState::new(attempt_budget);
1303 loop {
1304 if !retry.begin_submission() {
1305 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1306 }
1307 let request_id = request_ids
1308 .next_request_id()
1309 .map_err(|error| retry.request_id_failure(error))?;
1310 retry.note_request_id(&request_id);
1311 match self
1312 .apply_contract_migration(apply.request(request_id), metadata)
1313 .await
1314 {
1315 Ok(response) => return Ok(response),
1316 Err(error) => match retry.handle_failure(error) {
1317 RetryDecision::Retry => {}
1318 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1319 RetryDecision::Return(error) => return Err(error),
1320 },
1321 }
1322 }
1323 }
1324
1325 pub async fn start_application_installation_with_retry(
1327 &mut self,
1328 start: &StartApplicationInstallation,
1329 attempt_budget: AttemptBudget,
1330 metadata: &CallMetadata,
1331 ) -> Result<v1::StartApplicationInstallationResponse, ClientError> {
1332 let mut request_ids = SystemIdSource::new();
1333 let mut retry = RetryState::new(attempt_budget);
1334 loop {
1335 if !retry.begin_submission() {
1336 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1337 }
1338 let request_id = request_ids
1339 .next_request_id()
1340 .map_err(|error| retry.request_id_failure(error))?;
1341 retry.note_request_id(&request_id);
1342 match self
1343 .start_application_installation(start.request(request_id), metadata)
1344 .await
1345 {
1346 Ok(response) => return Ok(response),
1347 Err(error) => match retry.handle_failure(error) {
1348 RetryDecision::Retry => {}
1349 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1350 RetryDecision::Return(error) => return Err(error),
1351 },
1352 }
1353 }
1354 }
1355
1356 pub async fn start_application_export_with_retry(
1358 &mut self,
1359 start: &StartApplicationExport,
1360 attempt_budget: AttemptBudget,
1361 metadata: &CallMetadata,
1362 ) -> Result<v1::StartApplicationExportResponse, ClientError> {
1363 let mut request_ids = SystemIdSource::new();
1364 let mut retry = RetryState::new(attempt_budget);
1365 loop {
1366 if !retry.begin_submission() {
1367 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1368 }
1369 let request_id = request_ids
1370 .next_request_id()
1371 .map_err(|error| retry.request_id_failure(error))?;
1372 retry.note_request_id(&request_id);
1373 match self
1374 .start_application_export(start.request(request_id), metadata)
1375 .await
1376 {
1377 Ok(response) => return Ok(response),
1378 Err(error) => match retry.handle_failure(error) {
1379 RetryDecision::Retry => {}
1380 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1381 RetryDecision::Return(error) => return Err(error),
1382 },
1383 }
1384 }
1385 }
1386
1387 pub async fn get_application_export_page_with_retry(
1393 &mut self,
1394 operation_id: ApplicationExportOperationId,
1395 cursor: &[u8],
1396 max_rows: u16,
1397 attempt_budget: AttemptBudget,
1398 metadata: &CallMetadata,
1399 ) -> Result<v1::GetApplicationExportPageResponse, ClientError> {
1400 let mut request_ids = SystemIdSource::new();
1401 let mut retry = RetryState::new(attempt_budget);
1402 loop {
1403 if !retry.begin_submission() {
1404 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1405 }
1406 let request_id = request_ids
1407 .next_request_id()
1408 .map_err(|error| retry.request_id_failure(error))?;
1409 retry.note_request_id(&request_id);
1410 let request = v1::GetApplicationExportPageRequest {
1411 request_id: request_id.into_bytes().to_vec(),
1412 operation_id: operation_id.into_bytes().to_vec(),
1413 cursor: cursor.to_vec(),
1414 max_rows: u32::from(max_rows),
1415 };
1416 match self.get_application_export_page(request, metadata).await {
1417 Ok(response) => return Ok(response),
1418 Err(error) => match retry.handle_failure(error) {
1419 RetryDecision::Retry => {}
1420 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1421 RetryDecision::Return(error) => return Err(error),
1422 },
1423 }
1424 }
1425 }
1426
1427 pub async fn get_application_export_operation(
1429 &mut self,
1430 operation_id: ApplicationExportOperationId,
1431 metadata: &CallMetadata,
1432 ) -> Result<v1::GetApplicationExportResponse, ClientError> {
1433 let request_id = SystemIdSource::new()
1434 .next_request_id()
1435 .map_err(ClientError::IdentifierGeneration)?;
1436 self.get_application_export(
1437 v1::GetApplicationExportRequest {
1438 request_id: request_id.into_bytes().to_vec(),
1439 operation_id: operation_id.into_bytes().to_vec(),
1440 },
1441 metadata,
1442 )
1443 .await
1444 }
1445
1446 pub async fn cancel_application_export_with_retry(
1448 &mut self,
1449 operation_id: ApplicationExportOperationId,
1450 attempt_budget: AttemptBudget,
1451 metadata: &CallMetadata,
1452 ) -> Result<v1::CancelApplicationExportResponse, ClientError> {
1453 let mut request_ids = SystemIdSource::new();
1454 let mut retry = RetryState::new(attempt_budget);
1455 loop {
1456 if !retry.begin_submission() {
1457 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1458 }
1459 let request_id = request_ids
1460 .next_request_id()
1461 .map_err(|error| retry.request_id_failure(error))?;
1462 retry.note_request_id(&request_id);
1463 let request = v1::CancelApplicationExportRequest {
1464 request_id: request_id.into_bytes().to_vec(),
1465 operation_id: operation_id.into_bytes().to_vec(),
1466 };
1467 match self.cancel_application_export(request, metadata).await {
1468 Ok(response) => return Ok(response),
1469 Err(error) => match retry.handle_failure(error) {
1470 RetryDecision::Retry => {}
1471 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1472 RetryDecision::Return(error) => return Err(error),
1473 },
1474 }
1475 }
1476 }
1477
1478 pub async fn start_application_reimport_with_retry(
1480 &mut self,
1481 start: &StartApplicationReimport,
1482 attempt_budget: AttemptBudget,
1483 metadata: &CallMetadata,
1484 ) -> Result<v1::StartApplicationReimportResponse, ClientError> {
1485 let mut request_ids = SystemIdSource::new();
1486 let mut retry = RetryState::new(attempt_budget);
1487 loop {
1488 if !retry.begin_submission() {
1489 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1490 }
1491 let request_id = request_ids
1492 .next_request_id()
1493 .map_err(|error| retry.request_id_failure(error))?;
1494 retry.note_request_id(&request_id);
1495 match self
1496 .start_application_reimport(start.request(request_id), metadata)
1497 .await
1498 {
1499 Ok(response) => return Ok(response),
1500 Err(error) => match retry.handle_failure(error) {
1501 RetryDecision::Retry => {}
1502 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1503 RetryDecision::Return(error) => return Err(error),
1504 },
1505 }
1506 }
1507 }
1508
1509 pub async fn apply_application_reimport_page_with_retry(
1511 &mut self,
1512 campaign_id: ApplicationInstallationCampaignId,
1513 page: &v1::ApplicationExportPage,
1514 attempt_budget: AttemptBudget,
1515 metadata: &CallMetadata,
1516 ) -> Result<v1::ApplyApplicationReimportPageResponse, ClientError> {
1517 let mut request_ids = SystemIdSource::new();
1518 let mut retry = RetryState::new(attempt_budget);
1519 loop {
1520 if !retry.begin_submission() {
1521 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1522 }
1523 let request_id = request_ids
1524 .next_request_id()
1525 .map_err(|error| retry.request_id_failure(error))?;
1526 retry.note_request_id(&request_id);
1527 let request = v1::ApplyApplicationReimportPageRequest {
1528 request_id: request_id.into_bytes().to_vec(),
1529 campaign_id: campaign_id.into_bytes().to_vec(),
1530 page: Some(page.clone()),
1531 };
1532 match self
1533 .apply_application_reimport_page(request, metadata)
1534 .await
1535 {
1536 Ok(response) => return Ok(response),
1537 Err(error) => match retry.handle_failure(error) {
1538 RetryDecision::Retry => {}
1539 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1540 RetryDecision::Return(error) => return Err(error),
1541 },
1542 }
1543 }
1544 }
1545
1546 pub async fn get_application_reimport_operation(
1548 &mut self,
1549 campaign_id: ApplicationInstallationCampaignId,
1550 metadata: &CallMetadata,
1551 ) -> Result<v1::GetApplicationReimportResponse, ClientError> {
1552 let request_id = SystemIdSource::new()
1553 .next_request_id()
1554 .map_err(ClientError::IdentifierGeneration)?;
1555 self.get_application_reimport(
1556 v1::GetApplicationReimportRequest {
1557 request_id: request_id.into_bytes().to_vec(),
1558 campaign_id: campaign_id.into_bytes().to_vec(),
1559 },
1560 metadata,
1561 )
1562 .await
1563 }
1564
1565 pub async fn cancel_application_reimport_with_retry(
1567 &mut self,
1568 campaign_id: ApplicationInstallationCampaignId,
1569 attempt_budget: AttemptBudget,
1570 metadata: &CallMetadata,
1571 ) -> Result<v1::CancelApplicationReimportResponse, ClientError> {
1572 let mut request_ids = SystemIdSource::new();
1573 let mut retry = RetryState::new(attempt_budget);
1574 loop {
1575 if !retry.begin_submission() {
1576 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1577 }
1578 let request_id = request_ids
1579 .next_request_id()
1580 .map_err(|error| retry.request_id_failure(error))?;
1581 retry.note_request_id(&request_id);
1582 let request = v1::CancelApplicationReimportRequest {
1583 request_id: request_id.into_bytes().to_vec(),
1584 campaign_id: campaign_id.into_bytes().to_vec(),
1585 };
1586 match self.cancel_application_reimport(request, metadata).await {
1587 Ok(response) => return Ok(response),
1588 Err(error) => match retry.handle_failure(error) {
1589 RetryDecision::Retry => {}
1590 RetryDecision::RetryAfter(delay) => apply_overloaded_backoff(delay).await,
1591 RetryDecision::Return(error) => return Err(error),
1592 },
1593 }
1594 }
1595 }
1596
1597 pub async fn execute_generated<C: GeneratedCommand>(
1599 &mut self,
1600 command: &C,
1601 attempt_budget: AttemptBudget,
1602 metadata: &CallMetadata,
1603 ) -> Result<GeneratedExecution<C::Outcome>, GeneratedExecutionError> {
1604 let generic = command
1605 .idempotent_command()
1606 .map_err(GeneratedExecutionError::CommandShape)?;
1607 let command_name = generic.command_name().to_owned();
1608 let response = self
1609 .execute_with_retry(&generic, attempt_budget, metadata)
1610 .await
1611 .map_err(|error| contextualize_command_client_error(error, &command_name))
1612 .map_err(GeneratedExecutionError::Client)?;
1613 let outcome = command
1614 .decode_outcome(&response)
1615 .map_err(GeneratedExecutionError::CommandShape)?;
1616 Ok(GeneratedExecution { response, outcome })
1617 }
1618
1619 pub async fn execute_generated_with_recovery<C: GeneratedCommand>(
1622 &mut self,
1623 command: &C,
1624 attempt_budget: AttemptBudget,
1625 metadata: &CallMetadata,
1626 ) -> Result<GeneratedExecution<C::Outcome>, GeneratedExecutionError> {
1627 let generic = command
1628 .idempotent_command()
1629 .map_err(GeneratedExecutionError::CommandShape)?;
1630 let command_name = generic.command_name().to_owned();
1631 let response = match self
1632 .execute_with_retry(&generic, attempt_budget, metadata)
1633 .await
1634 {
1635 Ok(response) => response,
1636 Err(ClientError::OutcomeUnknown(_)) => {
1637 let request_id = crate::generate_request_id().map_err(|_| {
1638 GeneratedExecutionError::Client(ClientError::OutcomeUnknown(
1639 crate::OutcomeUnknown,
1640 ))
1641 })?;
1642 let request = command
1643 .outcome_request(request_id)
1644 .map_err(GeneratedExecutionError::CommandShape)?;
1645 let resolved = self
1646 .get_outcome(request, metadata)
1647 .await
1648 .map_err(GeneratedExecutionError::Client)?;
1649 match resolved.result {
1650 Some(v1::get_outcome_response::Result::Found(response)) => response,
1651 Some(v1::get_outcome_response::Result::NotFound(_)) | None => {
1652 return Err(GeneratedExecutionError::Client(
1653 ClientError::OutcomeUnknown(crate::OutcomeUnknown),
1654 ));
1655 }
1656 }
1657 }
1658 Err(error) => {
1659 return Err(GeneratedExecutionError::Client(
1660 contextualize_command_client_error(error, &command_name),
1661 ));
1662 }
1663 };
1664 let outcome = command
1665 .decode_outcome(&response)
1666 .map_err(GeneratedExecutionError::CommandShape)?;
1667 Ok(GeneratedExecution { response, outcome })
1668 }
1669}
1670
1671impl ExecuteRetryAttempt for RiffDbClient {
1672 async fn submit_execute_attempt(
1673 &mut self,
1674 request: v1::ExecuteCommandRequest,
1675 metadata: &CallMetadata,
1676 ) -> Result<v1::ExecuteCommandResponse, ClientError> {
1677 if let Some(session) = self.bounded_application_session.as_ref() {
1678 if !session.matches_metadata(metadata) {
1679 return Err(invalid_outbound());
1680 }
1681 return session.execute_command(request).await;
1682 }
1683 self.execute(request, metadata).await
1684 }
1685}
1686
1687impl NormalCapabilityRetryAttempt for RiffDbClient {
1688 async fn submit_normal_capability_attempt(
1689 &mut self,
1690 request: v1::CreateCapabilityRequest,
1691 metadata: &CallMetadata,
1692 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
1693 self.create_capability(request, metadata).await
1694 }
1695}
1696
1697impl BootstrapCapabilityRetryAttempt for RiffDbClient {
1698 async fn submit_bootstrap_capability_attempt(
1699 &mut self,
1700 request: v1::CreateCapabilityRequest,
1701 metadata: &BootstrapCallMetadata,
1702 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
1703 self.create_bootstrap_capability(request, metadata).await
1704 }
1705}
1706
1707pub(crate) async fn execute_retry_attempts<T: ExecuteRetryAttempt, S: RetryRequestIdSource>(
1708 attempts: &mut T,
1709 command: &IdempotentCommand,
1710 attempt_budget: AttemptBudget,
1711 metadata: &CallMetadata,
1712 request_ids: &mut S,
1713) -> Result<v1::ExecuteCommandResponse, ClientError> {
1714 let mut retry = RetryState::new(attempt_budget);
1715 loop {
1716 let Some(request) = next_retry_request(command, &mut retry, request_ids)? else {
1717 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1718 };
1719 match attempts.submit_execute_attempt(request, metadata).await {
1720 Ok(response) => return Ok(response),
1721 Err(error) => match retry.handle_failure(error) {
1722 RetryDecision::Retry => {}
1723 RetryDecision::RetryAfter(delay) => {
1724 apply_overloaded_backoff(delay).await;
1725 }
1726 RetryDecision::Return(error) => return Err(error),
1727 },
1728 }
1729 }
1730}
1731
1732async fn normal_capability_retry_attempts<
1733 T: NormalCapabilityRetryAttempt,
1734 S: RetryRequestIdSource,
1735>(
1736 attempts: &mut T,
1737 create: &NormalCapabilityCreateTemplate,
1738 attempt_budget: AttemptBudget,
1739 metadata: &CallMetadata,
1740 request_ids: &mut S,
1741) -> Result<v1::CreateCapabilityResponse, ClientError> {
1742 let mut retry = RetryState::new(attempt_budget);
1743 loop {
1744 let Some(request) = next_normal_create_request(create, &mut retry, request_ids)? else {
1745 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1746 };
1747 match attempts
1748 .submit_normal_capability_attempt(request, metadata)
1749 .await
1750 {
1751 Ok(response) => return Ok(response),
1752 Err(error) => match retry.handle_failure(error) {
1753 RetryDecision::Retry => {}
1754 RetryDecision::RetryAfter(delay) => {
1755 apply_overloaded_backoff(delay).await;
1756 }
1757 RetryDecision::Return(error) => return Err(error),
1758 },
1759 }
1760 }
1761}
1762
1763async fn bootstrap_capability_retry_attempts<
1764 T: BootstrapCapabilityRetryAttempt,
1765 S: RetryRequestIdSource,
1766>(
1767 attempts: &mut T,
1768 create: &BootstrapCapabilityCreateTemplate,
1769 attempt_budget: AttemptBudget,
1770 metadata: &BootstrapCallMetadata,
1771 request_ids: &mut S,
1772) -> Result<v1::CreateCapabilityResponse, ClientError> {
1773 let mut retry = RetryState::new(attempt_budget);
1774 loop {
1775 let Some(request) = next_bootstrap_create_request(create, &mut retry, request_ids)? else {
1776 return Err(ClientError::OutcomeUnknown(crate::OutcomeUnknown));
1777 };
1778 match attempts
1779 .submit_bootstrap_capability_attempt(request, metadata)
1780 .await
1781 {
1782 Ok(response) => return Ok(response),
1783 Err(error) => match retry.handle_failure(error) {
1784 RetryDecision::Retry => {}
1785 RetryDecision::RetryAfter(delay) => {
1786 apply_overloaded_backoff(delay).await;
1787 }
1788 RetryDecision::Return(error) => return Err(error),
1789 },
1790 }
1791 }
1792}
1793
1794fn next_retry_request<S: RetryRequestIdSource>(
1795 command: &IdempotentCommand,
1796 retry: &mut RetryState,
1797 request_ids: &mut S,
1798) -> Result<Option<v1::ExecuteCommandRequest>, ClientError> {
1799 if !retry.begin_submission() {
1800 return Ok(None);
1801 }
1802 request_ids
1803 .next_request_id()
1804 .map(|request_id| {
1805 retry.note_request_id(&request_id);
1806 Some(command.request(request_id))
1807 })
1808 .map_err(|error| retry.request_id_failure(error))
1809}
1810
1811fn next_normal_create_request<S: RetryRequestIdSource>(
1812 create: &NormalCapabilityCreateTemplate,
1813 retry: &mut RetryState,
1814 request_ids: &mut S,
1815) -> Result<Option<v1::CreateCapabilityRequest>, ClientError> {
1816 if !retry.begin_submission() {
1817 return Ok(None);
1818 }
1819 request_ids
1820 .next_request_id()
1821 .map(|request_id| {
1822 retry.note_request_id(&request_id);
1823 Some(create.request(request_id))
1824 })
1825 .map_err(|error| retry.request_id_failure(error))
1826}
1827
1828fn next_bootstrap_create_request<S: RetryRequestIdSource>(
1829 create: &BootstrapCapabilityCreateTemplate,
1830 retry: &mut RetryState,
1831 request_ids: &mut S,
1832) -> Result<Option<v1::CreateCapabilityRequest>, ClientError> {
1833 if !retry.begin_submission() {
1834 return Ok(None);
1835 }
1836 request_ids
1837 .next_request_id()
1838 .map(|request_id| {
1839 retry.note_request_id(&request_id);
1840 Some(create.request(request_id))
1841 })
1842 .map_err(|error| retry.request_id_failure(error))
1843}
1844
1845pub struct CommitNotificationStream {
1847 inner: Streaming<v1::CommitNotification>,
1848}
1849
1850pub struct EventConsumerResponseStream {
1852 inner: Streaming<v1::ConsumeEventStreamResponse>,
1853}
1854
1855pub struct LiveQueryUpdateStream {
1857 inner: Streaming<v1::LiveQueryUpdate>,
1858}
1859
1860impl LiveQueryUpdateStream {
1861 pub async fn message(&mut self) -> Result<Option<v1::LiveQueryUpdate>, ClientError> {
1863 self.inner.message().await.map_err(checked_status)
1864 }
1865}
1866
1867impl EventConsumerResponseStream {
1868 pub async fn message(&mut self) -> Result<Option<v1::ConsumeEventStreamResponse>, ClientError> {
1870 self.inner.message().await.map_err(checked_status)
1871 }
1872}
1873
1874impl CommitNotificationStream {
1875 pub async fn message(&mut self) -> Result<Option<v1::CommitNotification>, ClientError> {
1877 self.inner.message().await.map_err(checked_status)
1878 }
1879}
1880
1881pub struct GeneratedExecution<T> {
1883 response: v1::ExecuteCommandResponse,
1884 outcome: T,
1885}
1886
1887impl<T> GeneratedExecution<T> {
1888 #[must_use]
1890 pub const fn outcome(&self) -> &T {
1891 &self.outcome
1892 }
1893
1894 #[must_use]
1896 pub const fn response(&self) -> &v1::ExecuteCommandResponse {
1897 &self.response
1898 }
1899
1900 #[must_use]
1902 pub fn into_parts(self) -> (T, v1::ExecuteCommandResponse) {
1903 (self.outcome, self.response)
1904 }
1905}
1906
1907#[derive(Debug)]
1909pub enum GeneratedExecutionError {
1910 Client(ClientError),
1912 CommandShape(GeneratedCommandError),
1914}
1915
1916impl fmt::Display for GeneratedExecutionError {
1917 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1918 match self {
1919 Self::Client(error) => error.fmt(formatter),
1920 Self::CommandShape(error) => error.fmt(formatter),
1921 }
1922 }
1923}
1924
1925impl Error for GeneratedExecutionError {}
1926
1927fn validate_outbound<M: PublicMessage>(message: &M) -> Result<(), ClientError> {
1936 validate_public_message(message).map_err(|_| {
1937 ClientError::Protocol(ProtocolFailure::new(
1938 ProtocolFailureKind::InvalidOutboundMessage,
1939 ))
1940 })
1941}
1942
1943fn configure_projection_wait(
1944 message: &mut v1::QueryProjectionRequest,
1945 required_sequence: CommitSequence,
1946 maximum_wait: Duration,
1947) -> Result<(), ClientError> {
1948 message.required_sequence = Some(required_sequence.get());
1949 message.wait_nanos = u64::try_from(maximum_wait.as_nanos()).map_err(|_| invalid_outbound())?;
1950 validate_outbound(message)
1951}
1952
1953const fn invalid_inbound() -> ClientError {
1954 ClientError::Protocol(ProtocolFailure::new(
1955 ProtocolFailureKind::InvalidInboundMessage,
1956 ))
1957}
1958
1959const fn invalid_outbound() -> ClientError {
1960 ClientError::Protocol(ProtocolFailure::new(
1961 ProtocolFailureKind::InvalidOutboundMessage,
1962 ))
1963}
1964
1965#[cfg(test)]
1966mod tests {
1967 use std::collections::VecDeque;
1968 use std::future::Future;
1969 use std::sync::Arc;
1970 use std::sync::atomic::{AtomicUsize, Ordering};
1971 use std::task::{Context, Poll, Waker};
1972
1973 use super::*;
1974 use crate::BootstrapCredential;
1975 use crate::status::DetailsFreeStatus;
1976 use riffdb_errors::{PublicError, PublicErrorKind};
1977 use tonic_prost::prost::Message;
1978
1979 const BOOTSTRAP_TOKEN: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8";
1980
1981 struct FixedRequestIds {
1982 values: VecDeque<RequestId>,
1983 }
1984
1985 impl RetryRequestIdSource for FixedRequestIds {
1986 fn next_request_id(&mut self) -> Result<RequestId, crate::IdentifierGenerationError> {
1987 self.values
1988 .pop_front()
1989 .ok_or(crate::IdentifierGenerationError::EntropyUnavailable)
1990 }
1991 }
1992
1993 struct ScriptedRequestIds {
1994 results: VecDeque<Result<RequestId, crate::IdentifierGenerationError>>,
1995 calls: usize,
1996 }
1997
1998 impl RetryRequestIdSource for ScriptedRequestIds {
1999 fn next_request_id(&mut self) -> Result<RequestId, crate::IdentifierGenerationError> {
2000 self.calls += 1;
2001 self.results
2002 .pop_front()
2003 .unwrap_or(Err(crate::IdentifierGenerationError::EntropyUnavailable))
2004 }
2005 }
2006
2007 struct ExecuteAttempts {
2008 results: VecDeque<Result<v1::ExecuteCommandResponse, ClientError>>,
2009 requests: Vec<v1::ExecuteCommandRequest>,
2010 pending_once: bool,
2011 }
2012
2013 impl ExecuteAttempts {
2014 fn new(
2015 results: impl IntoIterator<Item = Result<v1::ExecuteCommandResponse, ClientError>>,
2016 ) -> Self {
2017 Self {
2018 results: results.into_iter().collect(),
2019 requests: Vec::new(),
2020 pending_once: false,
2021 }
2022 }
2023
2024 fn with_pending_once(mut self) -> Self {
2025 self.pending_once = true;
2026 self
2027 }
2028 }
2029
2030 impl ExecuteRetryAttempt for ExecuteAttempts {
2031 async fn submit_execute_attempt(
2032 &mut self,
2033 request: v1::ExecuteCommandRequest,
2034 _metadata: &CallMetadata,
2035 ) -> Result<v1::ExecuteCommandResponse, ClientError> {
2036 self.requests.push(request);
2037 if self.pending_once {
2038 let mut first_poll = true;
2039 std::future::poll_fn(|context| {
2040 if std::mem::take(&mut first_poll) {
2041 context.waker().wake_by_ref();
2042 Poll::Pending
2043 } else {
2044 Poll::Ready(())
2045 }
2046 })
2047 .await;
2048 self.pending_once = false;
2049 }
2050 self.results.pop_front().expect("scripted execute result")
2051 }
2052 }
2053
2054 struct NormalCapabilityAttempts {
2055 results: VecDeque<Result<v1::CreateCapabilityResponse, ClientError>>,
2056 requests: Vec<v1::CreateCapabilityRequest>,
2057 }
2058
2059 impl NormalCapabilityAttempts {
2060 fn new(
2061 results: impl IntoIterator<Item = Result<v1::CreateCapabilityResponse, ClientError>>,
2062 ) -> Self {
2063 Self {
2064 results: results.into_iter().collect(),
2065 requests: Vec::new(),
2066 }
2067 }
2068 }
2069
2070 impl NormalCapabilityRetryAttempt for NormalCapabilityAttempts {
2071 async fn submit_normal_capability_attempt(
2072 &mut self,
2073 request: v1::CreateCapabilityRequest,
2074 _metadata: &CallMetadata,
2075 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
2076 self.requests.push(request);
2077 self.results
2078 .pop_front()
2079 .expect("scripted normal-create result")
2080 }
2081 }
2082
2083 struct BootstrapCapabilityAttempts {
2084 results: VecDeque<Result<v1::CreateCapabilityResponse, ClientError>>,
2085 requests: Vec<v1::CreateCapabilityRequest>,
2086 credential_presentations: Vec<Vec<u8>>,
2087 }
2088
2089 impl BootstrapCapabilityAttempts {
2090 fn new(
2091 results: impl IntoIterator<Item = Result<v1::CreateCapabilityResponse, ClientError>>,
2092 ) -> Self {
2093 Self {
2094 results: results.into_iter().collect(),
2095 requests: Vec::new(),
2096 credential_presentations: Vec::new(),
2097 }
2098 }
2099 }
2100
2101 impl BootstrapCapabilityRetryAttempt for BootstrapCapabilityAttempts {
2102 async fn submit_bootstrap_capability_attempt(
2103 &mut self,
2104 request: v1::CreateCapabilityRequest,
2105 metadata: &BootstrapCallMetadata,
2106 ) -> Result<v1::CreateCapabilityResponse, ClientError> {
2107 let mut transport_request = Request::new(());
2108 metadata.apply(&mut transport_request);
2109 let presentation = transport_request
2110 .metadata()
2111 .get_bin("riffdb-bootstrap-token-bin")
2112 .expect("bootstrap credential metadata")
2113 .to_bytes()
2114 .expect("valid binary metadata")
2115 .to_vec();
2116 self.credential_presentations.push(presentation);
2117 self.requests.push(request);
2118 self.results
2119 .pop_front()
2120 .expect("scripted bootstrap-create result")
2121 }
2122 }
2123
2124 fn block_on_ready<T>(future: impl Future<Output = T>) -> T {
2125 let waker = Waker::noop();
2126 let mut context = Context::from_waker(waker);
2127 let mut future = std::pin::pin!(future);
2128 match future.as_mut().poll(&mut context) {
2129 Poll::Ready(output) => output,
2130 Poll::Pending => panic!("deterministic retry schedule unexpectedly yielded"),
2131 }
2132 }
2133
2134 #[derive(Default)]
2135 struct WakeCounter {
2136 wakes: AtomicUsize,
2137 }
2138
2139 impl std::task::Wake for WakeCounter {
2140 fn wake(self: Arc<Self>) {
2141 self.wakes.fetch_add(1, Ordering::SeqCst);
2142 }
2143
2144 fn wake_by_ref(self: &Arc<Self>) {
2145 self.wakes.fetch_add(1, Ordering::SeqCst);
2146 }
2147 }
2148
2149 fn block_on_deterministic_wakes<T>(future: impl Future<Output = T>) -> (T, usize) {
2150 let wake_counter = Arc::new(WakeCounter::default());
2151 let waker = Waker::from(Arc::clone(&wake_counter));
2152 let mut context = Context::from_waker(&waker);
2153 let mut future = std::pin::pin!(future);
2154 let mut pending_polls = 0;
2155 loop {
2156 match future.as_mut().poll(&mut context) {
2157 Poll::Ready(output) => return (output, pending_polls),
2158 Poll::Pending => {
2159 pending_polls += 1;
2160 assert!(
2161 pending_polls <= 1,
2162 "deterministic schedule exceeded its one pending poll"
2163 );
2164 assert_eq!(
2165 wake_counter.wakes.swap(0, Ordering::SeqCst),
2166 1,
2167 "each deterministic pending poll must arrange exactly one wake"
2168 );
2169 }
2170 }
2171 }
2172 }
2173
2174 fn scripted_ids(
2175 results: impl IntoIterator<Item = Result<RequestId, crate::IdentifierGenerationError>>,
2176 ) -> ScriptedRequestIds {
2177 ScriptedRequestIds {
2178 results: results.into_iter().collect(),
2179 calls: 0,
2180 }
2181 }
2182
2183 fn request_id(ordinal: u8) -> RequestId {
2184 RequestId::from_unix_milliseconds_and_random(u64::from(ordinal), [ordinal; 10])
2185 .expect("request ID")
2186 }
2187
2188 fn successful_execute_response() -> v1::ExecuteCommandResponse {
2189 v1::ExecuteCommandResponse {
2190 status: v1::execute_command_response::CompletionStatus::ExecutedReadOnly as i32,
2191 commit_sequence: 0,
2192 contract_version: 1,
2193 plan_hash: vec![7; 32],
2194 outcome_type: "Observed".to_owned(),
2195 outcome: Some(v1::Value {
2196 kind: Some(v1::value::Kind::RecordValue(v1::ValueRecord {
2197 fields: Vec::new(),
2198 })),
2199 }),
2200 provenance_uri: String::new(),
2201 durability_mode: String::new(),
2202 outcome_uri: None,
2203 history_incarnation: 1,
2204 }
2205 }
2206
2207 fn normal_token_unavailable_response(capability_id: &[u8]) -> v1::CreateCapabilityResponse {
2208 v1::CreateCapabilityResponse {
2209 result: Some(v1::create_capability_response::Result::Normal(
2210 v1::NormalCreateCapabilityResult {
2211 result: Some(
2212 v1::normal_create_capability_result::Result::AlreadyCreatedTokenUnavailable(
2213 v1::CapabilityIdentity {
2214 capability_id: capability_id.to_vec(),
2215 revision: 1,
2216 },
2217 ),
2218 ),
2219 },
2220 )),
2221 }
2222 }
2223
2224 fn bootstrap_replayed_response(capability_id: &[u8]) -> v1::CreateCapabilityResponse {
2225 v1::CreateCapabilityResponse {
2226 result: Some(v1::create_capability_response::Result::Bootstrap(
2227 v1::BootstrapCreateCapabilityResult {
2228 result: Some(v1::bootstrap_create_capability_result::Result::Replayed(
2229 v1::CapabilityTransition {
2230 identity: Some(v1::CapabilityIdentity {
2231 capability_id: capability_id.to_vec(),
2232 revision: 1,
2233 }),
2234 administration_sequence: 1,
2235 },
2236 )),
2237 },
2238 )),
2239 }
2240 }
2241
2242 fn transport_unavailable() -> ClientError {
2243 ClientError::DetailsFree(DetailsFreeStatus::TransportUnavailable)
2244 }
2245
2246 fn retryable_checked_error() -> ClientError {
2247 ClientError::Public(PublicError::concurrency_deadline_exceeded())
2248 }
2249
2250 fn public_outcome_unknown() -> ClientError {
2251 ClientError::Public(PublicError::outcome_unknown())
2252 }
2253
2254 fn terminal_checked_error() -> ClientError {
2255 ClientError::Public(PublicError::authorization_denied())
2256 }
2257
2258 fn assert_only_execute_request_id_changes(
2259 requests: &[v1::ExecuteCommandRequest],
2260 expected_ids: &[RequestId],
2261 ) {
2262 assert_eq!(requests.len(), expected_ids.len());
2263 let mut semantic_bodies = Vec::new();
2264 for (request, expected_id) in requests.iter().zip(expected_ids) {
2265 assert_eq!(request.request_id, expected_id.into_bytes());
2266 let mut semantic_body = request.clone();
2267 semantic_body.request_id.clear();
2268 semantic_bodies.push(semantic_body.encode_to_vec());
2269 }
2270 assert!(semantic_bodies.windows(2).all(|pair| pair[0] == pair[1]));
2271 }
2272
2273 fn assert_only_create_request_id_changes(
2274 requests: &[v1::CreateCapabilityRequest],
2275 expected_ids: &[RequestId],
2276 ) {
2277 assert_eq!(requests.len(), expected_ids.len());
2278 let mut semantic_bodies = Vec::new();
2279 for (request, expected_id) in requests.iter().zip(expected_ids) {
2280 assert_eq!(request.request_id, expected_id.into_bytes());
2281 let mut semantic_body = request.clone();
2282 semantic_body.request_id.clear();
2283 semantic_bodies.push(semantic_body.encode_to_vec());
2284 }
2285 assert!(semantic_bodies.windows(2).all(|pair| pair[0] == pair[1]));
2286 }
2287
2288 fn assert_authorization_denied<T>(result: Result<T, ClientError>) {
2289 assert!(matches!(
2290 result,
2291 Err(ClientError::Public(error))
2292 if error.kind() == PublicErrorKind::AuthorizationDenied
2293 ));
2294 }
2295
2296 fn assert_concurrency_deadline<T>(result: Result<T, ClientError>) {
2297 assert!(matches!(
2298 result,
2299 Err(ClientError::Public(error))
2300 if error.kind() == PublicErrorKind::ConcurrencyDeadlineExceeded
2301 ));
2302 }
2303
2304 fn assert_outcome_unknown<T>(result: Result<T, ClientError>) {
2305 assert!(matches!(result, Err(ClientError::OutcomeUnknown(_))));
2306 }
2307
2308 fn retry_input() -> v1::Value {
2309 v1::Value {
2310 kind: Some(v1::value::Kind::RecordValue(v1::ValueRecord {
2311 fields: vec![v1::ValueField {
2312 field_id: Some(1),
2313 name: String::new(),
2314 value: Some(v1::Value {
2315 kind: Some(v1::value::Kind::StringValue("same-key".to_owned())),
2316 }),
2317 }],
2318 })),
2319 }
2320 }
2321
2322 fn create_template_request(mode: v1::CapabilityCreateMode) -> v1::CreateCapabilityRequest {
2323 let permissions = if mode == v1::CapabilityCreateMode::Bootstrap {
2324 vec![v1::CapabilityPermission {
2325 permission: Some(
2326 v1::capability_permission::Permission::AdministerCapabilities(v1::Unit {}),
2327 ),
2328 }]
2329 } else {
2330 Vec::new()
2331 };
2332 v1::CreateCapabilityRequest {
2333 request_id: Vec::new(),
2334 mode: mode as i32,
2335 capability_id: RequestId::from_unix_milliseconds_and_random(1, [4; 10])
2336 .expect("capability ID")
2337 .into_bytes()
2338 .to_vec(),
2339 principal_id: "operator".to_owned(),
2340 actor_kind: v1::ActorKind::Human as i32,
2341 requested_lifetime_seconds: 60,
2342 audiences: vec!["riffdb-cli".to_owned()],
2343 grant: Some(v1::CapabilityGrant {
2344 tenant_scope: Some(v1::TenantScope {
2345 scope: Some(v1::tenant_scope::Scope::Global(v1::Unit {})),
2346 }),
2347 partition_scope: Some(v1::PartitionScope {
2348 scope: Some(v1::partition_scope::Scope::All(v1::Unit {})),
2349 }),
2350 permissions,
2351 field_visibility: Vec::new(),
2352 max_scan_rows: 1,
2353 approval_required: Vec::new(),
2354 row_policy: None,
2355 export: None,
2356 reimport: None,
2357 vector_inspection: None,
2358 }),
2359 }
2360 }
2361
2362 #[test]
2363 fn execute_helper_retries_checked_failure_with_fresh_id_and_stable_body() {
2364 let first_id = request_id(1);
2365 let second_id = request_id(2);
2366 let mut request_ids = scripted_ids([Ok(first_id), Ok(second_id)]);
2367 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2368 .expect("immutable command");
2369 let expected = successful_execute_response();
2370 let mut attempts =
2371 ExecuteAttempts::new([Err(retryable_checked_error()), Ok(expected.clone())]);
2372
2373 let result = block_on_ready(execute_retry_attempts(
2374 &mut attempts,
2375 &command,
2376 AttemptBudget::new(2).expect("budget"),
2377 &CallMetadata::default(),
2378 &mut request_ids,
2379 ));
2380
2381 assert_eq!(result.expect("second attempt succeeds"), expected);
2382 assert_eq!(request_ids.calls, 2);
2383 assert_only_execute_request_id_changes(&attempts.requests, &[first_id, second_id]);
2384 validate_public_message(&attempts.requests[0]).expect("first checked request");
2385 validate_public_message(&attempts.requests[1]).expect("second checked request");
2386 }
2387
2388 #[test]
2389 fn bootstrap_helper_retains_identity_body_and_token_across_retry() {
2390 let first_id = request_id(3);
2391 let second_id = request_id(4);
2392 let mut request_ids = scripted_ids([Ok(first_id), Ok(second_id)]);
2393 let template_request = create_template_request(v1::CapabilityCreateMode::Bootstrap);
2394 let capability_id = template_request.capability_id.clone();
2395 let create =
2396 BootstrapCapabilityCreateTemplate::new(template_request).expect("bootstrap template");
2397 let expected = bootstrap_replayed_response(&capability_id);
2398 let mut attempts = BootstrapCapabilityAttempts::new([
2399 Err(retryable_checked_error()),
2400 Ok(expected.clone()),
2401 ]);
2402 let metadata = BootstrapCallMetadata::new(
2403 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2404 );
2405
2406 let result = block_on_ready(bootstrap_capability_retry_attempts(
2407 &mut attempts,
2408 &create,
2409 AttemptBudget::new(2).expect("budget"),
2410 &metadata,
2411 &mut request_ids,
2412 ));
2413
2414 assert_eq!(result.expect("replayed bootstrap result"), expected);
2415 assert_eq!(request_ids.calls, 2);
2416 assert_only_create_request_id_changes(&attempts.requests, &[first_id, second_id]);
2417 assert!(
2418 attempts
2419 .requests
2420 .iter()
2421 .all(|request| request.capability_id == capability_id)
2422 );
2423 assert_eq!(
2424 attempts.credential_presentations,
2425 vec![BOOTSTRAP_TOKEN.as_bytes().to_vec(); 2]
2426 );
2427 validate_create_capability_exchange(&attempts.requests[1], &expected)
2428 .expect("checked bootstrap exchange");
2429 }
2430
2431 #[test]
2432 fn normal_helper_returns_token_unavailable_after_uncertainty_without_replacement() {
2433 let first_id = request_id(5);
2434 let second_id = request_id(6);
2435 let unused_third_id = request_id(7);
2436 let mut request_ids = scripted_ids([Ok(first_id), Ok(second_id), Ok(unused_third_id)]);
2437 let template_request = create_template_request(v1::CapabilityCreateMode::Normal);
2438 let capability_id = template_request.capability_id.clone();
2439 let create =
2440 NormalCapabilityCreateTemplate::new(template_request).expect("normal template");
2441 let expected = normal_token_unavailable_response(&capability_id);
2442 let mut attempts =
2443 NormalCapabilityAttempts::new([Err(transport_unavailable()), Ok(expected.clone())]);
2444
2445 let result = block_on_ready(normal_capability_retry_attempts(
2446 &mut attempts,
2447 &create,
2448 AttemptBudget::new(3).expect("budget"),
2449 &CallMetadata::default(),
2450 &mut request_ids,
2451 ));
2452
2453 assert_eq!(
2454 result.expect("terminal token-unavailable response"),
2455 expected
2456 );
2457 assert_eq!(request_ids.calls, 2);
2458 assert_eq!(request_ids.results.len(), 1);
2459 assert!(attempts.results.is_empty());
2460 assert_only_create_request_id_changes(&attempts.requests, &[first_id, second_id]);
2461 validate_create_capability_exchange(&attempts.requests[1], &expected)
2462 .expect("checked normal-create exchange");
2463 let Some(v1::create_capability_response::Result::Normal(normal)) = expected.result else {
2464 panic!("normal result family")
2465 };
2466 assert!(matches!(
2467 normal.result,
2468 Some(
2469 v1::normal_create_capability_result::Result::AlreadyCreatedTokenUnavailable(
2470 v1::CapabilityIdentity {
2471 capability_id: returned_id,
2472 revision: 1,
2473 }
2474 )
2475 ) if returned_id == capability_id
2476 ));
2477 }
2478
2479 #[test]
2480 fn every_helper_resolves_public_outcome_unknown_with_a_valid_terminal_result() {
2481 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2482 .expect("immutable command");
2483 let normal_request = create_template_request(v1::CapabilityCreateMode::Normal);
2484 let normal_capability_id = normal_request.capability_id.clone();
2485 let normal = NormalCapabilityCreateTemplate::new(normal_request).expect("normal template");
2486 let bootstrap_request = create_template_request(v1::CapabilityCreateMode::Bootstrap);
2487 let bootstrap_capability_id = bootstrap_request.capability_id.clone();
2488 let bootstrap =
2489 BootstrapCapabilityCreateTemplate::new(bootstrap_request).expect("bootstrap template");
2490 let bootstrap_metadata = BootstrapCallMetadata::new(
2491 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2492 );
2493 let budget = AttemptBudget::new(2).expect("budget");
2494
2495 let execute_request_ids = [request_id(35), request_id(36)];
2496 let mut execute_ids = scripted_ids(execute_request_ids.map(Ok));
2497 let execute_response = successful_execute_response();
2498 let mut execute =
2499 ExecuteAttempts::new([Err(public_outcome_unknown()), Ok(execute_response.clone())]);
2500 assert_eq!(
2501 block_on_ready(execute_retry_attempts(
2502 &mut execute,
2503 &command,
2504 budget,
2505 &CallMetadata::default(),
2506 &mut execute_ids,
2507 ))
2508 .expect("same-key command resolution"),
2509 execute_response
2510 );
2511 assert_eq!(execute_ids.calls, 2);
2512 assert_only_execute_request_id_changes(&execute.requests, &execute_request_ids);
2513
2514 let normal_request_ids = [request_id(37), request_id(38)];
2515 let mut normal_ids = scripted_ids(normal_request_ids.map(Ok));
2516 let normal_response = normal_token_unavailable_response(&normal_capability_id);
2517 let mut normal_attempts = NormalCapabilityAttempts::new([
2518 Err(public_outcome_unknown()),
2519 Ok(normal_response.clone()),
2520 ]);
2521 assert_eq!(
2522 block_on_ready(normal_capability_retry_attempts(
2523 &mut normal_attempts,
2524 &normal,
2525 budget,
2526 &CallMetadata::default(),
2527 &mut normal_ids,
2528 ))
2529 .expect("normal-create identity resolution"),
2530 normal_response
2531 );
2532 assert_eq!(normal_ids.calls, 2);
2533 assert_only_create_request_id_changes(&normal_attempts.requests, &normal_request_ids);
2534
2535 let bootstrap_request_ids = [request_id(39), request_id(40)];
2536 let mut bootstrap_ids = scripted_ids(bootstrap_request_ids.map(Ok));
2537 let bootstrap_response = bootstrap_replayed_response(&bootstrap_capability_id);
2538 let mut bootstrap_attempts = BootstrapCapabilityAttempts::new([
2539 Err(public_outcome_unknown()),
2540 Ok(bootstrap_response.clone()),
2541 ]);
2542 assert_eq!(
2543 block_on_ready(bootstrap_capability_retry_attempts(
2544 &mut bootstrap_attempts,
2545 &bootstrap,
2546 budget,
2547 &bootstrap_metadata,
2548 &mut bootstrap_ids,
2549 ))
2550 .expect("bootstrap identity resolution"),
2551 bootstrap_response
2552 );
2553 assert_eq!(bootstrap_ids.calls, 2);
2554 assert_only_create_request_id_changes(&bootstrap_attempts.requests, &bootstrap_request_ids);
2555 assert_eq!(
2556 bootstrap_attempts.credential_presentations,
2557 vec![BOOTSTRAP_TOKEN.as_bytes().to_vec(); 2]
2558 );
2559 }
2560
2561 #[test]
2562 fn execute_retry_state_survives_a_deterministic_pending_and_wake() {
2563 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2564 .expect("immutable command");
2565 let request_ids_expected = [request_id(41), request_id(42)];
2566 let mut request_ids = scripted_ids(request_ids_expected.map(Ok));
2567 let expected = successful_execute_response();
2568 let mut attempts =
2569 ExecuteAttempts::new([Err(public_outcome_unknown()), Ok(expected.clone())])
2570 .with_pending_once();
2571
2572 let (result, pending_polls) = block_on_deterministic_wakes(execute_retry_attempts(
2573 &mut attempts,
2574 &command,
2575 AttemptBudget::new(2).expect("budget"),
2576 &CallMetadata::default(),
2577 &mut request_ids,
2578 ));
2579
2580 assert_eq!(pending_polls, 1);
2581 assert_eq!(result.expect("retry after deterministic wake"), expected);
2582 assert_eq!(request_ids.calls, 2);
2583 assert_only_execute_request_id_changes(&attempts.requests, &request_ids_expected);
2584 }
2585
2586 #[test]
2587 fn every_helper_returns_terminal_checked_error_without_retry() {
2588 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2589 .expect("immutable command");
2590 let normal = NormalCapabilityCreateTemplate::new(create_template_request(
2591 v1::CapabilityCreateMode::Normal,
2592 ))
2593 .expect("normal template");
2594 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
2595 v1::CapabilityCreateMode::Bootstrap,
2596 ))
2597 .expect("bootstrap template");
2598 let bootstrap_metadata = BootstrapCallMetadata::new(
2599 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2600 );
2601 let budget = AttemptBudget::new(3).expect("budget");
2602
2603 let mut execute_ids = scripted_ids([Ok(request_id(8)), Ok(request_id(9))]);
2604 let mut execute = ExecuteAttempts::new([Err(terminal_checked_error())]);
2605 assert_authorization_denied(block_on_ready(execute_retry_attempts(
2606 &mut execute,
2607 &command,
2608 budget,
2609 &CallMetadata::default(),
2610 &mut execute_ids,
2611 )));
2612 assert_eq!((execute.requests.len(), execute_ids.calls), (1, 1));
2613
2614 let mut normal_ids = scripted_ids([Ok(request_id(10)), Ok(request_id(11))]);
2615 let mut normal_attempts = NormalCapabilityAttempts::new([Err(terminal_checked_error())]);
2616 assert_authorization_denied(block_on_ready(normal_capability_retry_attempts(
2617 &mut normal_attempts,
2618 &normal,
2619 budget,
2620 &CallMetadata::default(),
2621 &mut normal_ids,
2622 )));
2623 assert_eq!((normal_attempts.requests.len(), normal_ids.calls), (1, 1));
2624
2625 let mut bootstrap_ids = scripted_ids([Ok(request_id(12)), Ok(request_id(13))]);
2626 let mut bootstrap_attempts =
2627 BootstrapCapabilityAttempts::new([Err(terminal_checked_error())]);
2628 assert_authorization_denied(block_on_ready(bootstrap_capability_retry_attempts(
2629 &mut bootstrap_attempts,
2630 &bootstrap,
2631 budget,
2632 &bootstrap_metadata,
2633 &mut bootstrap_ids,
2634 )));
2635 assert_eq!(
2636 (bootstrap_attempts.requests.len(), bootstrap_ids.calls),
2637 (1, 1)
2638 );
2639 }
2640
2641 #[test]
2642 fn every_helper_keeps_uncertainty_when_a_later_error_is_terminal() {
2643 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2644 .expect("immutable command");
2645 let normal = NormalCapabilityCreateTemplate::new(create_template_request(
2646 v1::CapabilityCreateMode::Normal,
2647 ))
2648 .expect("normal template");
2649 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
2650 v1::CapabilityCreateMode::Bootstrap,
2651 ))
2652 .expect("bootstrap template");
2653 let bootstrap_metadata = BootstrapCallMetadata::new(
2654 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2655 );
2656 let budget = AttemptBudget::new(3).expect("budget");
2657
2658 let execute_request_ids = [request_id(29), request_id(30)];
2659 let mut execute_ids = scripted_ids(execute_request_ids.map(Ok));
2660 let mut execute =
2661 ExecuteAttempts::new([Err(public_outcome_unknown()), Err(terminal_checked_error())]);
2662 assert_outcome_unknown(block_on_ready(execute_retry_attempts(
2663 &mut execute,
2664 &command,
2665 budget,
2666 &CallMetadata::default(),
2667 &mut execute_ids,
2668 )));
2669 assert_eq!((execute.requests.len(), execute_ids.calls), (2, 2));
2670 assert_only_execute_request_id_changes(&execute.requests, &execute_request_ids);
2671
2672 let normal_request_ids = [request_id(31), request_id(32)];
2673 let mut normal_ids = scripted_ids(normal_request_ids.map(Ok));
2674 let mut normal_attempts = NormalCapabilityAttempts::new([
2675 Err(public_outcome_unknown()),
2676 Err(terminal_checked_error()),
2677 ]);
2678 assert_outcome_unknown(block_on_ready(normal_capability_retry_attempts(
2679 &mut normal_attempts,
2680 &normal,
2681 budget,
2682 &CallMetadata::default(),
2683 &mut normal_ids,
2684 )));
2685 assert_eq!((normal_attempts.requests.len(), normal_ids.calls), (2, 2));
2686 assert_only_create_request_id_changes(&normal_attempts.requests, &normal_request_ids);
2687
2688 let bootstrap_request_ids = [request_id(33), request_id(34)];
2689 let mut bootstrap_ids = scripted_ids(bootstrap_request_ids.map(Ok));
2690 let mut bootstrap_attempts = BootstrapCapabilityAttempts::new([
2691 Err(public_outcome_unknown()),
2692 Err(terminal_checked_error()),
2693 ]);
2694 assert_outcome_unknown(block_on_ready(bootstrap_capability_retry_attempts(
2695 &mut bootstrap_attempts,
2696 &bootstrap,
2697 budget,
2698 &bootstrap_metadata,
2699 &mut bootstrap_ids,
2700 )));
2701 assert_eq!(
2702 (bootstrap_attempts.requests.len(), bootstrap_ids.calls),
2703 (2, 2)
2704 );
2705 assert_only_create_request_id_changes(&bootstrap_attempts.requests, &bootstrap_request_ids);
2706 }
2707
2708 #[test]
2709 fn every_helper_preserves_checked_retry_error_on_known_exhaustion() {
2710 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2711 .expect("immutable command");
2712 let normal = NormalCapabilityCreateTemplate::new(create_template_request(
2713 v1::CapabilityCreateMode::Normal,
2714 ))
2715 .expect("normal template");
2716 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
2717 v1::CapabilityCreateMode::Bootstrap,
2718 ))
2719 .expect("bootstrap template");
2720 let bootstrap_metadata = BootstrapCallMetadata::new(
2721 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2722 );
2723 let budget = AttemptBudget::new(2).expect("budget");
2724
2725 let execute_request_ids = [request_id(14), request_id(15)];
2726 let mut execute_ids = scripted_ids(execute_request_ids.map(Ok));
2727 let mut execute = ExecuteAttempts::new([
2728 Err(retryable_checked_error()),
2729 Err(retryable_checked_error()),
2730 ]);
2731 assert_concurrency_deadline(block_on_ready(execute_retry_attempts(
2732 &mut execute,
2733 &command,
2734 budget,
2735 &CallMetadata::default(),
2736 &mut execute_ids,
2737 )));
2738 assert_eq!(execute_ids.calls, 2);
2739 assert_only_execute_request_id_changes(&execute.requests, &execute_request_ids);
2740
2741 let normal_request_ids = [request_id(16), request_id(17)];
2742 let mut normal_ids = scripted_ids(normal_request_ids.map(Ok));
2743 let mut normal_attempts = NormalCapabilityAttempts::new([
2744 Err(retryable_checked_error()),
2745 Err(retryable_checked_error()),
2746 ]);
2747 assert_concurrency_deadline(block_on_ready(normal_capability_retry_attempts(
2748 &mut normal_attempts,
2749 &normal,
2750 budget,
2751 &CallMetadata::default(),
2752 &mut normal_ids,
2753 )));
2754 assert_eq!(normal_ids.calls, 2);
2755 assert_only_create_request_id_changes(&normal_attempts.requests, &normal_request_ids);
2756
2757 let bootstrap_request_ids = [request_id(18), request_id(19)];
2758 let mut bootstrap_ids = scripted_ids(bootstrap_request_ids.map(Ok));
2759 let mut bootstrap_attempts = BootstrapCapabilityAttempts::new([
2760 Err(retryable_checked_error()),
2761 Err(retryable_checked_error()),
2762 ]);
2763 assert_concurrency_deadline(block_on_ready(bootstrap_capability_retry_attempts(
2764 &mut bootstrap_attempts,
2765 &bootstrap,
2766 budget,
2767 &bootstrap_metadata,
2768 &mut bootstrap_ids,
2769 )));
2770 assert_eq!(bootstrap_ids.calls, 2);
2771 assert_only_create_request_id_changes(&bootstrap_attempts.requests, &bootstrap_request_ids);
2772 }
2773
2774 #[test]
2775 fn every_helper_reports_unknown_outcome_on_transport_exhaustion() {
2776 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2777 .expect("immutable command");
2778 let normal = NormalCapabilityCreateTemplate::new(create_template_request(
2779 v1::CapabilityCreateMode::Normal,
2780 ))
2781 .expect("normal template");
2782 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
2783 v1::CapabilityCreateMode::Bootstrap,
2784 ))
2785 .expect("bootstrap template");
2786 let bootstrap_metadata = BootstrapCallMetadata::new(
2787 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2788 );
2789 let budget = AttemptBudget::new(2).expect("budget");
2790
2791 let execute_request_ids = [request_id(20), request_id(21)];
2792 let mut execute_ids = scripted_ids(execute_request_ids.map(Ok));
2793 let mut execute =
2794 ExecuteAttempts::new([Err(transport_unavailable()), Err(transport_unavailable())]);
2795 assert_outcome_unknown(block_on_ready(execute_retry_attempts(
2796 &mut execute,
2797 &command,
2798 budget,
2799 &CallMetadata::default(),
2800 &mut execute_ids,
2801 )));
2802 assert_eq!(execute_ids.calls, 2);
2803 assert_only_execute_request_id_changes(&execute.requests, &execute_request_ids);
2804
2805 let normal_request_ids = [request_id(22), request_id(23)];
2806 let mut normal_ids = scripted_ids(normal_request_ids.map(Ok));
2807 let mut normal_attempts = NormalCapabilityAttempts::new([
2808 Err(transport_unavailable()),
2809 Err(transport_unavailable()),
2810 ]);
2811 assert_outcome_unknown(block_on_ready(normal_capability_retry_attempts(
2812 &mut normal_attempts,
2813 &normal,
2814 budget,
2815 &CallMetadata::default(),
2816 &mut normal_ids,
2817 )));
2818 assert_eq!(normal_ids.calls, 2);
2819 assert_only_create_request_id_changes(&normal_attempts.requests, &normal_request_ids);
2820
2821 let bootstrap_request_ids = [request_id(24), request_id(25)];
2822 let mut bootstrap_ids = scripted_ids(bootstrap_request_ids.map(Ok));
2823 let mut bootstrap_attempts = BootstrapCapabilityAttempts::new([
2824 Err(transport_unavailable()),
2825 Err(transport_unavailable()),
2826 ]);
2827 assert_outcome_unknown(block_on_ready(bootstrap_capability_retry_attempts(
2828 &mut bootstrap_attempts,
2829 &bootstrap,
2830 budget,
2831 &bootstrap_metadata,
2832 &mut bootstrap_ids,
2833 )));
2834 assert_eq!(bootstrap_ids.calls, 2);
2835 assert_only_create_request_id_changes(&bootstrap_attempts.requests, &bootstrap_request_ids);
2836 }
2837
2838 #[test]
2839 fn every_helper_closes_request_id_failure_before_and_after_uncertainty() {
2840 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2841 .expect("immutable command");
2842 let normal = NormalCapabilityCreateTemplate::new(create_template_request(
2843 v1::CapabilityCreateMode::Normal,
2844 ))
2845 .expect("normal template");
2846 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
2847 v1::CapabilityCreateMode::Bootstrap,
2848 ))
2849 .expect("bootstrap template");
2850 let bootstrap_metadata = BootstrapCallMetadata::new(
2851 BootstrapCredential::new(BOOTSTRAP_TOKEN).expect("bootstrap credential"),
2852 );
2853 let budget = AttemptBudget::new(2).expect("budget");
2854 let id_failure = crate::IdentifierGenerationError::EntropyUnavailable;
2855
2856 let mut execute_ids = scripted_ids([Err(id_failure)]);
2857 let mut execute = ExecuteAttempts::new([]);
2858 assert!(matches!(
2859 block_on_ready(execute_retry_attempts(
2860 &mut execute,
2861 &command,
2862 budget,
2863 &CallMetadata::default(),
2864 &mut execute_ids,
2865 )),
2866 Err(ClientError::IdentifierGeneration(
2867 crate::IdentifierGenerationError::EntropyUnavailable
2868 ))
2869 ));
2870 assert!(execute.requests.is_empty());
2871 assert_eq!(execute_ids.calls, 1);
2872
2873 let mut normal_ids = scripted_ids([Err(id_failure)]);
2874 let mut normal_attempts = NormalCapabilityAttempts::new([]);
2875 assert!(matches!(
2876 block_on_ready(normal_capability_retry_attempts(
2877 &mut normal_attempts,
2878 &normal,
2879 budget,
2880 &CallMetadata::default(),
2881 &mut normal_ids,
2882 )),
2883 Err(ClientError::IdentifierGeneration(
2884 crate::IdentifierGenerationError::EntropyUnavailable
2885 ))
2886 ));
2887 assert!(normal_attempts.requests.is_empty());
2888 assert_eq!(normal_ids.calls, 1);
2889
2890 let mut bootstrap_ids = scripted_ids([Err(id_failure)]);
2891 let mut bootstrap_attempts = BootstrapCapabilityAttempts::new([]);
2892 assert!(matches!(
2893 block_on_ready(bootstrap_capability_retry_attempts(
2894 &mut bootstrap_attempts,
2895 &bootstrap,
2896 budget,
2897 &bootstrap_metadata,
2898 &mut bootstrap_ids,
2899 )),
2900 Err(ClientError::IdentifierGeneration(
2901 crate::IdentifierGenerationError::EntropyUnavailable
2902 ))
2903 ));
2904 assert!(bootstrap_attempts.requests.is_empty());
2905 assert_eq!(bootstrap_ids.calls, 1);
2906
2907 let mut execute_ids = scripted_ids([Ok(request_id(26)), Err(id_failure)]);
2908 let mut execute = ExecuteAttempts::new([Err(transport_unavailable())]);
2909 assert_outcome_unknown(block_on_ready(execute_retry_attempts(
2910 &mut execute,
2911 &command,
2912 budget,
2913 &CallMetadata::default(),
2914 &mut execute_ids,
2915 )));
2916 assert_eq!((execute.requests.len(), execute_ids.calls), (1, 2));
2917
2918 let mut normal_ids = scripted_ids([Ok(request_id(27)), Err(id_failure)]);
2919 let mut normal_attempts = NormalCapabilityAttempts::new([Err(transport_unavailable())]);
2920 assert_outcome_unknown(block_on_ready(normal_capability_retry_attempts(
2921 &mut normal_attempts,
2922 &normal,
2923 budget,
2924 &CallMetadata::default(),
2925 &mut normal_ids,
2926 )));
2927 assert_eq!((normal_attempts.requests.len(), normal_ids.calls), (1, 2));
2928
2929 let mut bootstrap_ids = scripted_ids([Ok(request_id(28)), Err(id_failure)]);
2930 let mut bootstrap_attempts =
2931 BootstrapCapabilityAttempts::new([Err(transport_unavailable())]);
2932 assert_outcome_unknown(block_on_ready(bootstrap_capability_retry_attempts(
2933 &mut bootstrap_attempts,
2934 &bootstrap,
2935 budget,
2936 &bootstrap_metadata,
2937 &mut bootstrap_ids,
2938 )));
2939 assert_eq!(
2940 (bootstrap_attempts.requests.len(), bootstrap_ids.calls),
2941 (1, 2)
2942 );
2943 }
2944
2945 #[test]
2946 fn adapter_minted_idempotency_survives_driver_retries_with_fresh_transport_ids() {
2947 let first_id = RequestId::from_unix_milliseconds_and_random(1, [1; 10]).expect("first ID");
2948 let second_id =
2949 RequestId::from_unix_milliseconds_and_random(2, [2; 10]).expect("second ID");
2950 let mut request_ids = FixedRequestIds {
2951 values: VecDeque::from([first_id, second_id]),
2952 };
2953 let command = IdempotentCommand::new("CreateBudget", Some(1), retry_input())
2954 .expect("immutable command");
2955 let mut retry = RetryState::new(AttemptBudget::new(2).expect("budget"));
2956
2957 let first = next_retry_request(&command, &mut retry, &mut request_ids)
2958 .expect("first request")
2959 .expect("first submission");
2960 assert!(matches!(
2961 retry.handle_failure(ClientError::DetailsFree(
2962 DetailsFreeStatus::TransportUnavailable
2963 )),
2964 RetryDecision::Retry
2965 ));
2966 let second = next_retry_request(&command, &mut retry, &mut request_ids)
2967 .expect("second request")
2968 .expect("second submission");
2969
2970 assert_eq!(first.request_id, first_id.into_bytes());
2971 assert_eq!(second.request_id, second_id.into_bytes());
2972 assert_ne!(first.request_id, second.request_id);
2973 assert_eq!(first.input, second.input);
2974 assert_eq!(
2975 first.input.as_ref().expect("first input").encode_to_vec(),
2976 second.input.as_ref().expect("second input").encode_to_vec()
2977 );
2978 assert_eq!(first.command_name, second.command_name);
2979 assert_eq!(
2980 first.expected_contract_version,
2981 second.expected_contract_version
2982 );
2983 assert!(request_ids.values.is_empty());
2984 assert!(matches!(
2985 retry.handle_failure(ClientError::DetailsFree(
2986 DetailsFreeStatus::TransportUnavailable
2987 )),
2988 RetryDecision::Return(ClientError::OutcomeUnknown(_))
2989 ));
2990 }
2991
2992 #[test]
2993 fn capability_retry_builders_preserve_body_and_apply_fresh_request_ids() {
2994 let first_id = RequestId::from_unix_milliseconds_and_random(4, [4; 10]).expect("first ID");
2995 let second_id =
2996 RequestId::from_unix_milliseconds_and_random(5, [5; 10]).expect("second ID");
2997 let mut request_ids = FixedRequestIds {
2998 values: VecDeque::from([first_id, second_id]),
2999 };
3000 let create = NormalCapabilityCreateTemplate::new(create_template_request(
3001 v1::CapabilityCreateMode::Normal,
3002 ))
3003 .expect("normal template");
3004 let mut retry = RetryState::new(AttemptBudget::new(2).expect("budget"));
3005
3006 let first = next_normal_create_request(&create, &mut retry, &mut request_ids)
3007 .expect("first request")
3008 .expect("first attempt");
3009 assert!(matches!(
3010 retry.handle_failure(ClientError::DetailsFree(
3011 DetailsFreeStatus::TransportUnavailable
3012 )),
3013 RetryDecision::Retry
3014 ));
3015 let second = next_normal_create_request(&create, &mut retry, &mut request_ids)
3016 .expect("second request")
3017 .expect("second attempt");
3018 assert_eq!(first.request_id, first_id.into_bytes());
3019 assert_eq!(second.request_id, second_id.into_bytes());
3020 let mut first_without_id = first;
3021 let mut second_without_id = second;
3022 first_without_id.request_id.clear();
3023 second_without_id.request_id.clear();
3024 assert_eq!(first_without_id, second_without_id);
3025 assert!(matches!(
3026 retry.handle_failure(ClientError::DetailsFree(
3027 DetailsFreeStatus::TransportUnavailable
3028 )),
3029 RetryDecision::Return(ClientError::OutcomeUnknown(_))
3030 ));
3031
3032 let bootstrap = BootstrapCapabilityCreateTemplate::new(create_template_request(
3033 v1::CapabilityCreateMode::Bootstrap,
3034 ))
3035 .expect("bootstrap template");
3036 let mut bootstrap_ids = FixedRequestIds {
3037 values: VecDeque::from([
3038 RequestId::from_unix_milliseconds_and_random(6, [6; 10]).expect("bootstrap ID")
3039 ]),
3040 };
3041 let mut bootstrap_retry = RetryState::new(AttemptBudget::new(1).expect("budget"));
3042 let request =
3043 next_bootstrap_create_request(&bootstrap, &mut bootstrap_retry, &mut bootstrap_ids)
3044 .expect("bootstrap request")
3045 .expect("bootstrap attempt");
3046 assert_eq!(request.mode, v1::CapabilityCreateMode::Bootstrap as i32);
3047 assert!(!request.request_id.is_empty());
3048 }
3049
3050 #[test]
3051 fn generated_execution_preserves_transport_metadata() {
3052 let response = v1::ExecuteCommandResponse {
3053 status: v1::execute_command_response::CompletionStatus::Committed as i32,
3054 commit_sequence: 9,
3055 contract_version: 1,
3056 plan_hash: vec![1; 32],
3057 outcome_type: "Created".to_owned(),
3058 outcome: Some(v1::Value {
3059 kind: Some(v1::value::Kind::RecordValue(v1::ValueRecord {
3060 fields: Vec::new(),
3061 })),
3062 }),
3063 provenance_uri: "riffdb://provenance/018f22e2-79b7-7cc3-a85f-250f0f80c78e".to_owned(),
3064 durability_mode: "sync".to_owned(),
3065 outcome_uri: None,
3066 history_incarnation: 1,
3067 };
3068 let execution = GeneratedExecution {
3069 response: response.clone(),
3070 outcome: 17_u32,
3071 };
3072 assert_eq!(execution.outcome(), &17);
3073 assert_eq!(execution.response(), &response);
3074 assert_eq!(execution.into_parts(), (17, response));
3075 }
3076
3077 #[test]
3078 fn projection_wait_maps_exact_total_nanoseconds_without_changing_query_identity() {
3079 let request_id = riffdb_types::RequestId::from_unix_milliseconds_and_random(1, [1; 10])
3080 .expect("request ID")
3081 .into_bytes()
3082 .to_vec();
3083 let contract = v1::ContractSelection {
3084 selection: Some(v1::contract_selection::Selection::Active(v1::Unit {})),
3085 };
3086 let mut request = v1::QueryProjectionRequest {
3087 request_id: request_id.clone(),
3088 contract: Some(contract.clone()),
3089 projection_id: 1,
3090 leading_components: Vec::new(),
3091 required_sequence: None,
3092 wait_nanos: 0,
3093 page: Some(v1::PageRequest {
3094 limit: Some(1),
3095 cursor: None,
3096 }),
3097 };
3098 let sequence = CommitSequence::new(7).expect("sequence");
3099 configure_projection_wait(&mut request, sequence, Duration::new(2, 345_678_901))
3100 .expect("bounded wait");
3101
3102 assert_eq!(request.request_id, request_id);
3103 assert_eq!(request.contract, Some(contract));
3104 assert_eq!(request.projection_id, 1);
3105 assert_eq!(request.required_sequence, Some(7));
3106 assert_eq!(request.wait_nanos, 2_345_678_901);
3107 }
3108
3109 #[test]
3110 fn projection_wait_rejects_the_public_thirty_second_overflow() {
3111 let mut request = v1::QueryProjectionRequest {
3112 request_id: riffdb_types::RequestId::from_unix_milliseconds_and_random(1, [2; 10])
3113 .expect("request ID")
3114 .into_bytes()
3115 .to_vec(),
3116 contract: Some(v1::ContractSelection {
3117 selection: Some(v1::contract_selection::Selection::Active(v1::Unit {})),
3118 }),
3119 projection_id: 1,
3120 leading_components: Vec::new(),
3121 required_sequence: None,
3122 wait_nanos: 0,
3123 page: Some(v1::PageRequest {
3124 limit: Some(1),
3125 cursor: None,
3126 }),
3127 };
3128 let error = configure_projection_wait(
3129 &mut request,
3130 CommitSequence::new(1).expect("sequence"),
3131 Duration::from_nanos(30_000_000_001),
3132 )
3133 .expect_err("over-limit wait");
3134 match error {
3135 ClientError::Protocol(failure) => {
3136 assert_eq!(failure.kind(), ProtocolFailureKind::InvalidOutboundMessage);
3137 }
3138 other => panic!("expected InvalidOutboundMessage, got {other:?}"),
3139 }
3140 }
3141
3142 #[test]
3146 fn invalid_unary_outbound_is_protocol_not_transport_unavailable() {
3147 use crate::status::{carries_uncertainty, is_retryable};
3148
3149 let request = v1::QueryProjectionRequest {
3150 request_id: riffdb_types::RequestId::from_unix_milliseconds_and_random(1, [3; 10])
3151 .expect("request ID")
3152 .into_bytes()
3153 .to_vec(),
3154 contract: Some(v1::ContractSelection {
3155 selection: Some(v1::contract_selection::Selection::Active(v1::Unit {})),
3156 }),
3157 projection_id: 1,
3158 leading_components: Vec::new(),
3159 required_sequence: Some(1),
3160 wait_nanos: 30_000_000_001,
3161 page: Some(v1::PageRequest {
3162 limit: Some(1),
3163 cursor: None,
3164 }),
3165 };
3166 let error =
3167 validate_outbound(&request).expect_err("invalid outbound must fail before any RPC");
3168 match &error {
3169 ClientError::Protocol(failure) => {
3170 assert_eq!(
3171 failure.kind(),
3172 ProtocolFailureKind::InvalidOutboundMessage,
3173 "encoder-only rejection would surface as TransportUnavailable"
3174 );
3175 }
3176 other => panic!("expected Protocol(InvalidOutboundMessage), got {other:?}"),
3177 }
3178 assert!(!is_retryable(&error));
3179 assert!(!carries_uncertainty(&error));
3180 }
3181
3182 #[test]
3185 fn max_canonical_value_inside_the_command_frame_validates_locally() {
3186 use riffdb_types::MAX_CANONICAL_DOCUMENT_BYTES;
3187
3188 let payload_len = MAX_CANONICAL_DOCUMENT_BYTES - 32;
3189 let input = v1::Value {
3190 kind: Some(v1::value::Kind::RecordValue(v1::ValueRecord {
3191 fields: vec![v1::ValueField {
3192 field_id: Some(1),
3193 name: "b".to_owned(),
3194 value: Some(v1::Value {
3195 kind: Some(v1::value::Kind::BytesValue(vec![0x61; payload_len])),
3196 }),
3197 }],
3198 })),
3199 };
3200 let request_id = riffdb_types::RequestId::from_unix_milliseconds_and_random(1, [4; 10])
3201 .expect("request ID");
3202 let execute = v1::ExecuteCommandRequest {
3203 request_id: request_id.into_bytes().to_vec(),
3204 command_name: "CreateTicket".to_owned(),
3205 expected_contract_version: Some(1),
3206 input: Some(input.clone()),
3207 };
3208 validate_outbound(&execute).expect("bounded command frame");
3209
3210 let command = IdempotentCommand::new("CreateTicket", Some(1), input)
3211 .expect("max-bound input is a legal command document");
3212 let built = command.request(request_id);
3213 validate_outbound(&built).expect("retry path retains the bounded command frame");
3214 }
3215}
3216
3217#[cfg(test)]
3218mod history_incarnation_tests {
3219 use riffdb_proto::v1;
3220 use tonic::transport::Endpoint;
3221
3222 use super::{RiffDbClient, merge_observed_history_incarnation};
3223
3224 #[test]
3226 fn merge_prefers_explicit_over_remembered() {
3227 assert_eq!(merge_observed_history_incarnation(None, Some(7)), Some(7));
3228 assert_eq!(
3229 merge_observed_history_incarnation(Some(3), Some(7)),
3230 Some(3)
3231 );
3232 assert_eq!(merge_observed_history_incarnation(None, None), None);
3233 assert_eq!(merge_observed_history_incarnation(Some(3), None), Some(3));
3234 }
3235
3236 #[test]
3238 fn client_and_mcp_track_observed_history_incarnation_across_commit_and_event_reads() {
3239 let runtime = tokio::runtime::Builder::new_current_thread()
3240 .enable_time()
3241 .build()
3242 .expect("current-thread runtime");
3243 let _guard = runtime.enter();
3244 let mut client =
3245 RiffDbClient::from_channel(Endpoint::from_static("http://[::1]:1").connect_lazy());
3246 assert_eq!(client.observed_history_incarnation(), None);
3247
3248 client.remember_history_incarnation_from_get(&v1::GetCommitResponse {
3249 result: None,
3250 history_incarnation: 4,
3251 });
3252 assert_eq!(client.observed_history_incarnation(), Some(4));
3253
3254 let get_req = client.apply_observed_history_incarnation_get(v1::GetCommitRequest {
3255 request_id: vec![1; 16],
3256 commit_sequence: 1,
3257 observed_history_incarnation: None,
3258 });
3259 assert_eq!(get_req.observed_history_incarnation, Some(4));
3260
3261 let explicit_get = client.apply_observed_history_incarnation_get(v1::GetCommitRequest {
3262 request_id: vec![1; 16],
3263 commit_sequence: 1,
3264 observed_history_incarnation: Some(2),
3265 });
3266 assert_eq!(explicit_get.observed_history_incarnation, Some(2));
3267
3268 client.remember_history_incarnation_from_scan(&v1::ScanCommitsResponse {
3269 page: Some(v1::CommitPage {
3270 items: Vec::new(),
3271 next_cursor: None,
3272 observed_fence: None,
3273 history_incarnation: 5,
3274 }),
3275 });
3276 assert_eq!(client.observed_history_incarnation(), Some(5));
3277
3278 let scan_req = client.apply_observed_history_incarnation_scan(v1::ScanCommitsRequest {
3279 request_id: vec![1; 16],
3280 page: None,
3281 observed_history_incarnation: None,
3282 });
3283 assert_eq!(scan_req.observed_history_incarnation, Some(5));
3284
3285 let sub_req =
3286 client.apply_observed_history_incarnation_subscribe(v1::SubscribeCommitsRequest {
3287 request_id: vec![1; 16],
3288 after_sequence: None,
3289 maximum_lifetime_nanos: 1_000,
3290 observed_history_incarnation: None,
3291 });
3292 assert_eq!(sub_req.observed_history_incarnation, Some(5));
3293
3294 client.remember_history_incarnation_from_event_page(Some(&v1::EventPage {
3295 items: Vec::new(),
3296 next_cursor: Vec::new(),
3297 observed_upper: None,
3298 disposition: 0,
3299 history_incarnation: 9,
3300 }));
3301 assert_eq!(client.observed_history_incarnation(), Some(9));
3302
3303 let replay_req = client.apply_observed_history_incarnation_replay(v1::ReplayEventsRequest {
3304 request_id: vec![1; 16],
3305 selection: None,
3306 after_event_id: None,
3307 page: None,
3308 observed_history_incarnation: 0,
3309 });
3310 assert_eq!(replay_req.observed_history_incarnation, 9);
3311
3312 let explicit_replay =
3313 client.apply_observed_history_incarnation_replay(v1::ReplayEventsRequest {
3314 request_id: vec![1; 16],
3315 selection: None,
3316 after_event_id: None,
3317 page: None,
3318 observed_history_incarnation: 3,
3319 });
3320 assert_eq!(explicit_replay.observed_history_incarnation, 3);
3321
3322 let tail_req = client.apply_observed_history_incarnation_tail(v1::TailEventsRequest {
3323 request_id: vec![1; 16],
3324 selection: None,
3325 after_event_id: None,
3326 page: None,
3327 maximum_wait_nanos: 1_000,
3328 observed_history_incarnation: 0,
3329 });
3330 assert_eq!(tail_req.observed_history_incarnation, 9);
3331 }
3332}