1pub(crate) use operations::interrupt_operation;
2mod acquisition;
3pub use acquisition::AcquisitionAuthorization;
4mod catalogue;
5mod catalogue_search;
6mod datafile_materialization;
7mod dataset_transform;
8mod disclosure;
9mod governance;
10mod lifecycle;
11mod redcap_upload;
12mod semantic;
13mod semantic_batch;
14mod semantic_batch_admission;
15mod upload;
16use semantic_batch::{ResolvedEntityBatch, ResolvedRelationBatch};
17mod semantic_instances;
18mod semantic_lifecycle;
19mod semantic_search;
20mod semantic_tags;
21pub use governance::GovernedAssetDuo;
22pub use lifecycle::{GovernedDeleteOutcome, GovernedDeletePlan, GovernedDeleteRequest};
23pub use semantic_lifecycle::GovernedSemanticDeleteOutcome;
24mod operations;
25mod study_context;
26pub use operations::{
27 OperationAdmission, OperationCancellationPolicy, OperationContext, OperationControl,
28 OperationSessionAuthority, OperationSubmission,
29};
30mod dataset_attempt;
31mod metadata_admission;
32use dataset_attempt::{DatasetMaterializationAttempt, WriterAuthority};
33
34use crate::sql_source::{SqlSourceConnection, value_type_name};
35use crate::{
36 AddDelegateCustodianRequest, AddDomainRequest, AddEntityAssetLinkRequest,
37 AddEntityDatasetLinkRequest, AddEntityInstanceRequest, AddEntityRelationRequest,
38 AddEntityRequest, AddRelationAssetLinkRequest, AddRelationDatasetLinkRequest,
39 AddRelationInstanceRequest, AddStudyDomainRequest, AddStudyEntityInstanceMappingRequest,
40 AddStudyRelationInstanceMappingRequest, AddStudyRequest, AddVariableRequest,
41 AddVocabularyRequest, AnalyzeSqlSourceRequest, AppError, ArchiveDeleteAuthorization,
42 ArchiveDeleteAuthorizationStatus, ArchiveDeleteCleanupDisposition,
43 ArchiveDeleteDependencySummary, ArchiveDeletePlan, ArchiveDeletePlanningResult,
44 ArchiveDeleteRetryState, ArchiveDeleteStagePreview, ArchiveDeleteValidationError,
45 ArchiveDeleteVersionTarget, ArchiveStudyForDeletionRequest, ArchiveStudyForDeletionResult,
46 ArchivedDataFileRecord, ArchivedDatasetFileRecord, AssetDuoRestrictionSource,
47 AssetVersionCatalog, AssetVersionDuoRestrictionClear, AssetVersionDuoRestrictionList,
48 AssetVersionDuoRestrictionReplacement, ClearAssetVersionDuoRestrictionsRequest,
49 CreateAssetRequest, CreateAssetVersionRequest, CreateNewAssetRequest,
50 CreateNewAssetVersionRequest, DataFileMetadata, DataFileMetadataByIdRequest,
51 DataFileMetadataRequest, DataFileSearchEntry, DataFileSearchResult, DataFileToDatasetRequest,
52 DataStoreSession, DatasetExportFormat, DatasetListItem, DatasetMaterialization,
53 DatasetMetadataByIdRequest, DatasetMetadataResult, DatasetMetadataSelector, DatasetPreview,
54 DatasetPreviewColumn, DatasetSearchEntry, DatasetSearchResult, DatasetSqlTransformRequest,
55 DatasetVariableMetadata, DatasetVariableRegistrationRequest, DatasetVariablesMetadata,
56 DatastoreContentCount, DatastoreContentCounts, DeleteArchivedStudyRequest,
57 DeleteArchivedStudyResult, DeleteAssetRequest, DeleteAssetResult, DeleteAssetVersionResult,
58 DeleteDataFileAssetVersionRequest, DeleteDataFileAssetVersionResult,
59 DeleteDatasetAssetVersionRequest, DeleteDatasetAssetVersionResult,
60 DeriveDatasetFromManagedDataFileRequest, DerivedManagedDataFileDataset, DomainAddResult,
61 DomainListResult, DomainSelector, EffectiveAssetVersionDuoRestriction,
62 EffectiveAssetVersionDuoRestrictionList, EnsureEntityInstancesFromDatasetRequest,
63 EnsureRelationInstancesFromDatasetRequest, EnsureStudyEntityInstanceRequest,
64 EnsureStudyRelationInstanceRequest, EntityAssetLinkList, EntityAssetLinkReadback,
65 EntityDatasetLinkList, EntityDatasetLinkReadback, EntityInstanceDatasetBatchConflict,
66 EntityInstanceDatasetBatchCounts, EntityInstanceDatasetBatchRejectedRow,
67 EntityInstanceDatasetBatchResult, EntityInstanceMapping, EntityInstanceMappingList,
68 EntityInstanceMappingReadback, EntityRelationSelector, EntitySearchEntry, EntitySearchResult,
69 EntitySelector, ExecuteSemanticDeleteRequest, ExecuteSemanticDeleteResult,
70 ExportDataFileRequest, ExportDatasetRequest, ExportDatasetToPathRequest,
71 ExportStudyDataFileRequest, ExportedDataFile, ExportedDataset, FileToDatasetRequest,
72 GetAssetRequest, GetDatasetVariablesRequest, GetDomainRequest, GetEntityDatasetLinkRequest,
73 GetEntityInstanceRequest, GetEntityRelationRequest, GetEntityRequest,
74 GetRelationDatasetLinkRequest, GetRelationInstanceRequest, GetStudyDatasetRequest,
75 GetStudyEntityInstanceMappingRequest, GetStudyRegistrationRequest,
76 GetStudyRelationInstanceMappingRequest, GetStudyRequest, GetTagsRequest, GetVariableRequest,
77 GetVocabularyRequest, GrantStudyAccessRequest, IngestDataFileRequest, IngestDatasetFileRequest,
78 IngestFileContent, IngestFileRequest, IngestFileSource, IngestedDatasetFile, IngestedFileAsset,
79 LakeTableDropDisposition, LinkDatasetVersionEntitiesRequest,
80 LinkDatasetVersionRelationInstancesRequest, ListAssetVersionDuoRestrictionsRequest,
81 ListDomainsRequest, ListEffectiveAssetVersionDuoRestrictionsRequest, ListEntitiesRequest,
82 ListEntityAssetLinksRequest, ListEntityDatasetLinksRequest, ListEntityInstanceDatasetsRequest,
83 ListEntityInstancesRequest, ListEntityRelationsRequest, ListRelationAssetLinksRequest,
84 ListRelationDatasetLinksRequest, ListRelationInstancesRequest, ListStudiesRequest,
85 ListStudyAccessGrantsRequest, ListStudyAssetsRequest, ListStudyCustodiansRequest,
86 ListStudyDataFilesRequest, ListStudyDatasetsRequest, ListStudyDuoRestrictionsRequest,
87 ListStudyEntityInstanceMappingsRequest, ListStudyRelationInstanceMappingsRequest,
88 ListStudyTransformationsRequest, ListVariablesRequest, ListVocabulariesRequest,
89 ListVocabularyMappingsRequest, ManagedDataFileDatasetMaterialization,
90 OfflineRedcapIngestRequest, OpenDuckDbSourceRequest, OpenMsSqlSourceRequest,
91 OpenPostgresSourceRequest, OpenSqliteSourceRequest, PlanArchiveDeleteRequest,
92 PlanSemanticDeleteRequest, PrepareStudyArchiveForDeletionRequest,
93 PreparedArchiveDatafileIdentity, PreparedStudyArchiveForDeletionResult,
94 PreviewDatasetByIdRequest, PreviewStudyDatasetRequest, ReadDatasetMetadataRequest,
95 RecordExportRequest, RecordTransformationRequest, RedcapIngestResult,
96 RegisterDatasetVariablesRequest, RegisterDomainRequest, RegisterDomainStudyRequest,
97 RegisterEntityRelationRequest, RegisterEntityRequest, RegisterNewDomainRequest,
98 RegisterNewStudyRequest, RegisterStudyRequest, RegisterVariableRequest,
99 RegisterVocabularyItemRequest, RegisterVocabularyMappingRequest, RegisterVocabularyRequest,
100 RegisteredDatasetVariable, RegisteredVariable, RegisteredVocabulary, RelationAssetLinkList,
101 RelationAssetLinkReadback, RelationDatasetLinkList, RelationDatasetLinkReadback,
102 RelationInstanceDatasetBatchConflict, RelationInstanceDatasetBatchCounts,
103 RelationInstanceDatasetBatchMissingEndpoint, RelationInstanceDatasetBatchRejectedRow,
104 RelationInstanceDatasetBatchResult, RelationInstanceMapping, RelationInstanceMappingList,
105 RelationInstanceMappingReadback, RelationSearchEntry, RelationSearchResult,
106 RemoveDelegateCustodianRequest, ReplaceAssetVersionDuoRestrictionsRequest,
107 ReplaceStudyDuoRestrictionsRequest, RevokeStudyAccessRequest, SearchDataFilesRequest,
108 SearchDatasetsRequest, SearchEntitiesRequest, SearchRelationsRequest, SearchStudiesRequest,
109 SearchVariablesRequest, SemanticDeleteAuthorization, SemanticDeleteAuthorizationStatus,
110 SemanticDeleteDependency, SemanticDeleteDependencyStatus, SemanticDeleteDependencySummary,
111 SemanticDeleteDisposition, SemanticDeleteExecutionDisposition, SemanticDeletePlan,
112 SemanticDeletePlanningResult, SemanticDeleteTarget, SemanticDeleteTargetIdentity,
113 SemanticDeleteTargetKind, SemanticDeleteValidationError, SetTagsRequest, SqlSourceAnalysis,
114 SqlToDatasetRequest, StudyAccessGrant, StudyAccessGrantList, StudyAccessRevocation,
115 StudyAddResult, StudyArchiveManifest, StudyArchiveRegistration, StudyArchiveSummary,
116 StudyArchiveTarget, StudyCustodian, StudyCustodianList, StudyDataFileEntry, StudyDatasetList,
117 StudyDatasetMetadata, StudyDelegateCustodianRemoval, StudyDuoRestrictionList,
118 StudyDuoRestrictionReplacement, StudyEntityInstanceRegistration, StudyListResult,
119 StudyPrimaryCustodianshipTransfer, StudyRegistration, StudyRelationInstanceRegistration,
120 StudySearchEntry, StudySearchResult, StudySelector, TargetTags,
121 TransferPrimaryCustodianshipRequest, TransformAssetsRequest, TransformationSummary,
122 UpdateVariableRequest, VariableSearchEntry, VariableSearchResult, VariableSelector,
123 VocabularyItemReference, VocabularyMappingDetail, VocabularyMappingSummary,
124 VocabularyMappingSummaryPeer, VocabularyMappingSummaryRequest, VocabularySelector,
125 WithdrawDatasetVersionRequest, WithdrawDatasetVersionResult,
126};
127use ahri_tre_core::{
128 ArchiveDeletePolicy, ArchiveMode, AssetRepository, CoreError, DataFileSearchQuery,
129 DatasetSearchQuery, DefaultArchiveDeletePolicy, DefaultProvenancePolicy,
130 DefaultStudyGovernancePolicy, DeletePolicy, DeleteRequest, DeleteTarget, DomainRepository,
131 EntityRepository, EntitySearchQuery, GovernanceContext, RelationSearchQuery,
132 StudyDomainRepository, StudyGovernancePolicy, StudyGovernanceRepository, StudyRepository,
133 StudySearchQuery, TagAttachmentTarget, TagRepository, TransformationRepository,
134 VariableRepository, VariableSearchQuery, VocabularyRepository,
135};
136use ahri_tre_lake::{
137 DataFileCompression, DataFileEncryption, DatasetFileFormat, DatasetFileReadOptions,
138 DatasetTableRelation, DeleteDataFileRequest, DuckLakeAdapter, ExportDatasetTableRequest,
139 ImportedColumn, LoadDatasetTableFromFileRequest, LoadDatasetTableFromSqlRequest,
140 LoadRedcapFormDatasetRequest, PreviewDatasetTableRequest, RedcapEavChoice, RedcapEavField,
141 RedcapSyntheticColumn, StageDataFileStreamRequest, StagedDataFile,
142};
143use ahri_tre_protocol::search::MAX_SUMMARY_TAGS;
144use ahri_tre_redcap::{
145 RedcapChoice, RedcapDictionary, RedcapForm, RedcapProjectInfo, RedcapVariableSpec,
146 parse_project_info_file, sanitize_ncname_fragment,
147};
148use ahri_tre_security::UriRedactionMode;
149use ahri_tre_tabular::{
150 AhriTreArrowMetadata, AhriTreFieldMetadata, FieldMetadataProposal, metadata_keys,
151 read_ipc_schema, read_parquet_schema,
152};
153use ahri_tre_types::{AccessMode, DomainRecord, StudyId, StudyRecord};
154use chrono::Utc;
155use reqwest::Url;
156use reqwest::blocking::Client;
157use reqwest::redirect::{Attempt, Policy};
158use std::collections::BTreeMap;
159use std::env;
160use std::fs;
161use std::io::{Cursor, Read};
162use std::path::{Path, PathBuf};
163use std::process::Command;
164use std::sync::Arc;
165use std::sync::LazyLock;
166#[cfg(test)]
167use std::sync::Mutex;
168use std::time::Duration as StdDuration;
169
170const HTTPS_INGEST_TIMEOUT: StdDuration = StdDuration::from_secs(30);
171const HTTPS_INGEST_MAX_REDIRECTS: usize = 5;
172const HTTPS_INGEST_MAX_BYTES: u64 = 100 * 1024 * 1024;
173static FILE_INGEST_WORKFLOW_LOCK: LazyLock<Arc<tokio::sync::Mutex<()>>> =
174 LazyLock::new(|| Arc::new(tokio::sync::Mutex::new(())));
175
176struct FileIngestWorkflowRequest {
177 pub classification: ahri_tre_types::IngestClassification,
178 study_id: StudyId,
179 asset_id: Option<ahri_tre_types::AssetId>,
180 asset_name: ahri_tre_types::NcName,
181 version_id: Option<ahri_tre_types::VersionId>,
182 content: IngestFileContent,
183 lake_root: String,
184 edam_format: String,
185 validate_as_csv: bool,
186 compress: bool,
187 encrypt: Option<bool>,
188 description: Option<String>,
189 agent_instructions: Option<String>,
190 version_note: Option<String>,
191 transformation: ahri_tre_types::NewTransformationRecord,
192}
193
194struct FileIngestRollbackContext<'a> {
195 asset_id: ahri_tre_types::AssetId,
196 asset_preexisted: bool,
197 version_id: ahri_tre_types::VersionId,
198 tags_before_ingest: &'a [ahri_tre_types::TagRecord],
199 asset_tags_before_ingest: &'a [ahri_tre_types::TagRecord],
200 requested_asset_tags: &'a [String],
201 requested_version_tags: &'a [String],
202}
203
204struct StagedFileGuard {
205 staged: Option<StagedDataFile>,
206 lake: DuckLakeAdapter,
207 workflow_guard: Option<tokio::sync::OwnedMutexGuard<()>>,
208}
209
210impl StagedFileGuard {
211 fn staged(&self) -> &StagedDataFile {
212 self.staged
213 .as_ref()
214 .expect("staged Datafile guard is armed")
215 }
216
217 fn disarm(&mut self) {
218 self.staged = None;
219 self.workflow_guard = None;
220 }
221
222 fn commit(mut self) -> StagedDataFile {
223 let staged = self.staged.take().expect("staged Datafile guard is armed");
224 self.workflow_guard = None;
225 staged
226 }
227}
228
229impl Drop for StagedFileGuard {
230 fn drop(&mut self) {
231 let Some(staged) = self.staged.take() else {
232 return;
233 };
234 let lake = self.lake.clone();
235 let workflow_guard = self.workflow_guard.take();
236 if let Ok(handle) = tokio::runtime::Handle::try_current() {
237 let _cleanup = handle.spawn_blocking(move || {
238 rollback_uncommitted_datafile(lake, staged, workflow_guard);
239 });
240 } else {
241 rollback_uncommitted_datafile(lake, staged, workflow_guard);
242 }
243 }
244}
245
246fn rollback_uncommitted_datafile(
247 lake: DuckLakeAdapter,
248 staged: StagedDataFile,
249 workflow_guard: Option<tokio::sync::OwnedMutexGuard<()>>,
250) {
251 if let Err(error) = lake.delete_datafile(&DeleteDataFileRequest {
252 datafile: staged.datafile,
253 }) {
254 tracing::error!(%error, "could not roll back an uncommitted Datafile ingest");
255 }
256 drop(workflow_guard);
257}
258
259#[cfg(test)]
260static TEST_HTTPS_ROOT_CERTIFICATE_DER: Mutex<Option<Vec<u8>>> = Mutex::new(None);
261#[cfg(test)]
262static TEST_HTTPS_MAX_BYTES: Mutex<Option<u64>> = Mutex::new(None);
263#[cfg(test)]
264static TEST_HTTPS_TIMEOUT: Mutex<Option<StdDuration>> = Mutex::new(None);
265#[cfg(test)]
266static TEST_HTTPS_FIXTURE_LOCK: Mutex<()> = Mutex::new(());
267
268pub type AppService = ScopedAppService<'static>;
269
270#[derive(Clone)]
271pub struct ScopedAppService<'repository> {
272 semantic_repository: Option<ahri_tre_pgmeta::PgMetadataRepository<'repository>>,
273 datastore_id: Option<uuid::Uuid>,
274 registrations: Arc<dyn ahri_tre_core::CatalogueRegistrationRepository + 'repository>,
275 pub domains: Arc<dyn DomainRepository + 'repository>,
276 pub studies: Arc<dyn StudyRepository + 'repository>,
277 pub study_domains: Arc<dyn StudyDomainRepository + 'repository>,
278 pub study_governance: Arc<dyn StudyGovernanceRepository + 'repository>,
279 pub assets: Arc<dyn AssetRepository + 'repository>,
280 pub variables: Arc<dyn VariableRepository + 'repository>,
281 pub vocabularies: Arc<dyn VocabularyRepository + 'repository>,
282 pub entities: Arc<dyn EntityRepository + 'repository>,
283 pub transformations: Arc<dyn TransformationRepository + 'repository>,
284 pub tags: Arc<dyn TagRepository + 'repository>,
285}
286
287pub type AppRepositories = ScopedAppRepositories<'static>;
288
289#[derive(Clone)]
290pub struct ScopedAppRepositories<'repository> {
291 pub registrations: Arc<dyn ahri_tre_core::CatalogueRegistrationRepository + 'repository>,
292 pub domains: Arc<dyn DomainRepository + 'repository>,
293 pub studies: Arc<dyn StudyRepository + 'repository>,
294 pub study_domains: Arc<dyn StudyDomainRepository + 'repository>,
295 pub study_governance: Arc<dyn StudyGovernanceRepository + 'repository>,
296 pub assets: Arc<dyn AssetRepository + 'repository>,
297 pub variables: Arc<dyn VariableRepository + 'repository>,
298 pub vocabularies: Arc<dyn VocabularyRepository + 'repository>,
299 pub entities: Arc<dyn EntityRepository + 'repository>,
300 pub transformations: Arc<dyn TransformationRepository + 'repository>,
301 pub tags: Arc<dyn TagRepository + 'repository>,
302}
303
304pub struct AuthenticatedDataStoreSession {
305 pub session: DataStoreSession,
306 pub authenticated_tre_user: Option<String>,
307}
308
309impl<'repository> From<crate::session::ScopedSessionMetadataRepositories<'repository>>
310 for ScopedAppRepositories<'repository>
311{
312 fn from(repositories: crate::session::ScopedSessionMetadataRepositories<'repository>) -> Self {
313 Self {
314 registrations: repositories.registrations,
315 domains: repositories.domains,
316 studies: repositories.studies,
317 study_domains: repositories.study_domains,
318 study_governance: repositories.study_governance,
319 assets: repositories.assets,
320 variables: repositories.variables,
321 vocabularies: repositories.vocabularies,
322 entities: repositories.entities,
323 transformations: repositories.transformations,
324 tags: repositories.tags,
325 }
326 }
327}
328
329struct VariableCommandFields {
330 domain: DomainSelector,
331 name: String,
332 value_type: Option<String>,
333 value_format: Option<String>,
334 key_role: Option<String>,
335 description: Option<String>,
336 ontology_namespace: Option<String>,
337 ontology_class: Option<String>,
338 vocabulary: Option<String>,
339 vocabulary_items: Option<String>,
340 existing: Option<ahri_tre_types::VariableRecord>,
341 force_metadata_updates: bool,
342}
343
344impl AppService {
345 pub fn from_datastore_session(session: &DataStoreSession) -> Result<Self, AppError> {
346 let repositories = session.metadata_repositories()?;
347 let mut service = Self::new(repositories.into());
348 service.semantic_repository = session.dataset_admission_repository();
349 service.datastore_id = session
350 .runtime()
351 .identity_binding
352 .as_ref()
353 .map(|binding| binding.datastore_id);
354 Ok(service)
355 }
356
357 pub async fn list_session_domains_observed(
359 context: &ahri_tre_observability::CorrelationContext,
360 session: &DataStoreSession,
361 request: ListDomainsRequest,
362 ) -> Result<DomainListResult, AppError> {
363 use ahri_tre_observability::{FailureCategory, Outcome, Stage};
364 let span = context.span(Stage::Application);
365 let result = async {
366 let mut service = Self::from_datastore_session(session)?;
367 if let Some(repository) = session.dataset_admission_repository() {
368 service.domains = Arc::new(repository.with_correlation(span.context()));
369 }
370 let result = service.list_domains(request).await;
371 session.observations().record(
372 ahri_tre_protocol::diagnostics::DiagnosticComponent::Metadata,
373 ahri_tre_protocol::diagnostics::DiagnosticObservationOrigin::Workflow,
374 result.is_ok(),
375 );
376 result
377 }
378 .await;
379 let (outcome, category) = match &result {
380 Ok(_) => (Outcome::Success, None),
381 Err(AppError::Infrastructure(_)) => {
382 (Outcome::Unavailable, Some(FailureCategory::Dependency))
383 }
384 Err(_) => (Outcome::Rejected, None),
385 };
386 span.finish(outcome, category);
387 result
388 }
389
390 pub async fn open_configured_datastore_session<R: crate::DatastoreSecretResolver>(
393 &self,
394 request: crate::OpenConfiguredDatastoreRequest,
395 resolver: &R,
396 ) -> Result<DataStoreSession, AppError> {
397 crate::open_configured_datastore(request, resolver)
398 }
399
400 pub async fn datastore_content_counts(&self) -> Result<DatastoreContentCounts, AppError> {
401 let studies = self.studies.list_studies().await;
402 let domains = self.domains.list_domains().await;
403
404 let (studies_count, assets, datasets, datafiles) = match studies {
405 Ok(studies) => {
406 let mut asset_counts = CountAccumulator::default();
407 let mut dataset_counts = CountAccumulator::default();
408 let mut datafile_counts = CountAccumulator::default();
409
410 for study in &studies {
411 match self.assets.list_assets_for_study(study.study_id).await {
412 Ok(assets) => {
413 asset_counts.add(assets.len());
414 for asset in assets {
415 match self.assets.list_datasets_for_asset(asset.asset_id).await {
416 Ok(datasets) => dataset_counts.add(datasets.len()),
417 Err(error) => dataset_counts.record_error(error),
418 }
419 match self.assets.list_datafiles_for_asset(asset.asset_id).await {
420 Ok(datafiles) => datafile_counts.add(datafiles.len()),
421 Err(error) => datafile_counts.record_error(error),
422 }
423 }
424 }
425 Err(error) => {
426 asset_counts.record_error(error);
427 dataset_counts.record_unavailable();
428 datafile_counts.record_unavailable();
429 }
430 }
431 }
432
433 (
434 DatastoreContentCount::available(studies.len() as u64),
435 asset_counts.into_count(),
436 dataset_counts.into_count(),
437 datafile_counts.into_count(),
438 )
439 }
440 Err(CoreError::NotImplemented(_)) => (
441 DatastoreContentCount::unsupported(),
442 DatastoreContentCount::unavailable(),
443 DatastoreContentCount::unavailable(),
444 DatastoreContentCount::unavailable(),
445 ),
446 Err(_) => (
447 DatastoreContentCount::unavailable(),
448 DatastoreContentCount::unavailable(),
449 DatastoreContentCount::unavailable(),
450 DatastoreContentCount::unavailable(),
451 ),
452 };
453
454 let (variables, vocabularies, entities) = match domains {
455 Ok(domains) => {
456 let mut variable_counts = CountAccumulator::default();
457 let mut vocabulary_counts = CountAccumulator::default();
458 let mut entity_counts = CountAccumulator::default();
459
460 for domain in &domains {
461 match self.variables.list_domain_variables(domain.domain_id).await {
462 Ok(variables) => variable_counts.add(variables.len()),
463 Err(error) => variable_counts.record_error(error),
464 }
465 match self.vocabularies.list_vocabularies(domain.domain_id).await {
466 Ok(vocabularies) => vocabulary_counts.add(vocabularies.len()),
467 Err(error) => vocabulary_counts.record_error(error),
468 }
469 match self.entities.list_entities(domain.domain_id).await {
470 Ok(entities) => entity_counts.add(entities.len()),
471 Err(error) => entity_counts.record_error(error),
472 }
473 }
474
475 (
476 variable_counts.into_count(),
477 vocabulary_counts.into_count(),
478 entity_counts.into_count(),
479 )
480 }
481 Err(CoreError::NotImplemented(_)) => (
482 DatastoreContentCount::unavailable(),
483 DatastoreContentCount::unavailable(),
484 DatastoreContentCount::unavailable(),
485 ),
486 Err(_) => (
487 DatastoreContentCount::unavailable(),
488 DatastoreContentCount::unavailable(),
489 DatastoreContentCount::unavailable(),
490 ),
491 };
492
493 let transformations = match self.transformations.list_transformations().await {
494 Ok(transformations) => DatastoreContentCount::available(transformations.len() as u64),
495 Err(CoreError::NotImplemented(_)) => DatastoreContentCount::unsupported(),
496 Err(_) => DatastoreContentCount::unavailable(),
497 };
498
499 Ok(DatastoreContentCounts {
500 studies: studies_count,
501 assets,
502 datasets,
503 datafiles,
504 entities,
505 variables,
506 vocabularies,
507 transformations,
508 })
509 }
510
511 pub async fn file_to_dataset(
512 &self,
513 session: &mut DataStoreSession,
514 request: FileToDatasetRequest,
515 ) -> Result<DatasetMaterialization, AppError> {
516 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Ingest
517 {
518 return Err(AppError::Validation(
519 "file-to-dataset requires an ingest transformation".to_string(),
520 ));
521 }
522
523 let source_path = PathBuf::from(&request.input_path);
524 if !source_path.is_file() {
525 return Err(AppError::Validation(format!(
526 "dataset source file not found: {}",
527 source_path.display()
528 )));
529 }
530 let source_format =
531 resolve_dataset_file_format(Some(&source_path), request.input_format.as_deref())?;
532 let read_options = dataset_file_read_options(
533 request.header,
534 request.delimiter,
535 request.null_strings.clone(),
536 request.json_format.clone(),
537 request.sheet.clone(),
538 );
539 let mut attempt = self
540 .prepare_dataset_attempt(
541 session,
542 PrepareDatasetVersionRequest {
543 output_intent: None,
544 inherit_risk: false,
545 classification: request.classification.clone(),
546 study_id: request.study_id,
547 dataset_asset_id: request.dataset_asset_id,
548 dataset_name: request.dataset_name,
549 dataset_version_id: request.dataset_version_id,
550 description: request.description,
551 agent_instructions: request.agent_instructions,
552 version_note: request.version_note,
553 created_by: request.transformation.created_by.clone(),
554 },
555 )
556 .await?;
557 let prepared = attempt.prepared.clone();
558 let result = async {
559 let lake_root = session.runtime().lake.data_path.clone();
560 let lake = DuckLakeAdapter::new(&lake_root);
561 let source = if source_format == DatasetFileFormat::ArrowIpc {
562 Some(
563 lake.stage_dataset_source_file(&attempt.scratch, &source_path, source_format)
564 .map_err(|error| infrastructure_error("stage Dataset source", error))?,
565 )
566 } else {
567 None
568 };
569 let source_path = source
570 .as_ref()
571 .map(|source| source.path().to_path_buf())
572 .unwrap_or(source_path);
573 let load_result = lake
574 .load_dataset_table_from_file_reserved(
575 session.lake_connection(),
576 &mut attempt.authority,
577 &LoadDatasetTableFromFileRequest {
578 study_id: request.study_id,
579 dataset_name: prepared.asset.name.clone(),
580 major: prepared.version.major,
581 minor: prepared.version.minor,
582 patch: prepared.version.patch,
583 source_path,
584 format: source_format,
585 options: read_options,
586 },
587 )
588 .map_err(|error| infrastructure_error("load external file into DuckLake", error));
589 let source_cleanup = source.map(|source| source.cleanup()).transpose();
590 let loaded = match load_result {
591 Ok(loaded) => loaded,
592 Err(error) => {
593 source_cleanup
594 .map_err(|error| infrastructure_error("clean Dataset source", error))?;
595 return Err(error);
596 }
597 };
598 let relation = loaded.relation.clone();
599 let dataset_record = ahri_tre_types::DatasetRecord {
600 dataset_id: prepared.version.version_id,
601 };
602
603 let transformation = enrich_transformation_source(&request.transformation);
604 let metadata_result = async {
605 source_cleanup
606 .map_err(|error| infrastructure_error("clean Dataset source", error))?;
607 let (catalog, dataset, lineage, variables) = self
608 .save_dataset_metadata_with_options(
609 session,
610 &mut attempt,
611 dataset_record,
612 &transformation,
613 &[],
614 DatasetMetadataRegistration {
615 redcap_dictionary: None,
616 variable_registrations: request
617 .variable_registrations
618 .into_iter()
619 .map(Into::into)
620 .collect(),
621 loaded_column_names: Some(&loaded.column_names),
622 force_metadata_updates: false,
623 },
624 )
625 .await?;
626 Ok::<_, AppError>((catalog, dataset, lineage, variables))
627 }
628 .await;
629
630 let (catalog, dataset, lineage, variables) = metadata_result?;
631
632 Ok(DatasetMaterialization {
633 catalog,
634 dataset,
635 lineage,
636 lake_relation: relation.qualified_name(),
637 lake_schema: relation.schema_name,
638 lake_table: relation.table_name,
639 row_count: loaded.row_count,
640 column_count: loaded.column_count,
641 variables,
642 metadata_warnings: Vec::new(),
643 })
644 }
645 .await;
646 attempt.finish(session, result)
647 }
648
649 pub async fn derive_dataset_from_managed_datafile(
650 &self,
651 session: &mut DataStoreSession,
652 request: DeriveDatasetFromManagedDataFileRequest,
653 ) -> Result<DerivedManagedDataFileDataset, AppError> {
654 self.derive_dataset_with_lifecycle(
655 session,
656 request,
657 &mut datafile_materialization::SynchronousDatafileLifecycle,
658 )
659 .await
660 }
661
662 async fn derive_dataset_with_lifecycle(
663 &self,
664 session: &mut DataStoreSession,
665 request: DeriveDatasetFromManagedDataFileRequest,
666 lifecycle: &mut impl datafile_materialization::DatafileLifecycle,
667 ) -> Result<DerivedManagedDataFileDataset, AppError> {
668 if (request.replace || request.new_version.is_some()) && lifecycle.output_intent().is_none()
669 {
670 return Err(AppError::Validation(
671 "managed Datafile derivation does not support replacement or an explicit output version"
672 .to_string(),
673 ));
674 }
675 let (study, domain, source, input_format) = datafile_materialization::observed(
676 lifecycle.observation(),
677 ahri_tre_observability::Stage::Metadata,
678 async {
679 let study = self
680 .get_study(GetStudyRequest {
681 study: request.study,
682 })
683 .await?
684 .ok_or_else(|| {
685 AppError::NotFound("Dataset derivation Study not found".to_string())
686 })?;
687 let domain = self
688 .get_domain(GetDomainRequest {
689 domain: request.metadata_domain,
690 })
691 .await?
692 .ok_or_else(|| {
693 AppError::NotFound("Dataset metadata Domain not found".to_string())
694 })?;
695 let (source_study, catalog, pinned) =
696 self.resolve_catalogue_asset(request.source.asset).await?;
697 if source_study.study_id != study.study.study_id {
698 return Err(AppError::Validation(
699 "This materialization requires source and Output in the same Study".into(),
700 ));
701 }
702 if catalog.asset.asset_type != ahri_tre_types::AssetType::File {
703 return Err(AppError::Validation("A Datafile source is required".into()));
704 }
705 let version =
706 self.catalogue_version(&catalog, pinned, request.source.version.as_deref())?;
707 let datafile = self
708 .assets
709 .get_datafile(version.version_id)
710 .await
711 .map_err(|error| core_error("lookup source Datafile", error))?
712 .ok_or_else(|| AppError::NotFound("Datafile version was not found".into()))?;
713 let source = DataFileMetadata {
714 catalog,
715 version,
716 datafile,
717 };
718 let input_format = managed_datafile_input_format(
719 source.datafile.edam_format.as_str(),
720 request.input_format.as_deref(),
721 )?;
722 Ok((study, domain, source, input_format))
723 },
724 )
725 .await?;
726 lifecycle.resolved_source(&study.study, &source);
727 let dataset = self
728 .datafile_to_dataset_with_lifecycle(
729 session,
730 DataFileToDatasetRequest {
731 study_id: study.study.study_id,
732 metadata_domain_id: Some(domain.domain_id),
733 dataset_asset_id: None,
734 dataset_name: request.dataset_name,
735 dataset_version_id: None,
736 source_datafile_id: source.datafile.datafile_id,
737 input_format: Some(input_format),
738 header: request.header,
739 delimiter: request.delimiter,
740 null_strings: request.null_strings,
741 json_format: request.json_format,
742 sheet: request.sheet,
743 description: request.description,
744 agent_instructions: None,
745 version_note: request.version_note,
746 transformation: request.transformation,
747 strict_metadata: false,
748 sample_rows: request.sample_rows,
749 vocabulary_threshold: request.vocabulary_threshold,
750 force_metadata_updates: request.force_metadata_updates,
751 variable_registrations: Vec::new(),
752 },
753 lifecycle,
754 )
755 .await?;
756 Ok(DerivedManagedDataFileDataset {
757 study: study.study,
758 materialization: ManagedDataFileDatasetMaterialization { source, dataset },
759 })
760 }
761
762 pub async fn ingest_dataset_file(
763 &self,
764 session: &mut DataStoreSession,
765 request: IngestDatasetFileRequest,
766 ) -> Result<IngestedDatasetFile, AppError> {
767 if request.ingest_transformation.transformation_type
768 != ahri_tre_types::TransformationType::Ingest
769 {
770 return Err(AppError::Validation(
771 "dataset table source staging requires an ingest transformation".to_string(),
772 ));
773 }
774 if request.materialize_transformation.transformation_type
775 != ahri_tre_types::TransformationType::Transform
776 {
777 return Err(AppError::Validation(
778 "dataset table materialization requires a transform transformation".to_string(),
779 ));
780 }
781
782 let source_asset_name = request
783 .source_asset_name
784 .unwrap_or_else(|| derived_source_file_asset_name(&request.dataset_name));
785 let source_format = request.input_format.clone();
786 let lake_root = session.runtime().lake.data_path.clone();
787 let encrypt_source = crate::resolve_file_asset_encryption(request.encrypt_source);
788 let source_file = self
789 .ingest_file_for_session(
790 session,
791 IngestFileRequest {
792 classification: request.classification.clone(),
793 study_id: request.study_id,
794 asset_id: None,
795 asset_name: source_asset_name,
796 version_id: None,
797 source: request.source,
798 lake_root,
799 edam_format: source_format.clone(),
800 compress: request.compress_source,
801 encrypt: Some(encrypt_source),
802 description: request.description.clone(),
803 agent_instructions: request.agent_instructions.clone(),
804 version_note: request.version_note.clone(),
805 transformation: request.ingest_transformation,
806 },
807 )
808 .await?;
809
810 let dataset = self
811 .datafile_to_dataset(
812 session,
813 DataFileToDatasetRequest {
814 study_id: request.study_id,
815 metadata_domain_id: None,
816 dataset_asset_id: request.dataset_asset_id,
817 dataset_name: request.dataset_name,
818 dataset_version_id: request.dataset_version_id,
819 source_datafile_id: source_file.datafile.datafile_id,
820 input_format: source_format,
821 header: request.header,
822 delimiter: request.delimiter,
823 null_strings: request.null_strings,
824 json_format: request.json_format,
825 sheet: request.sheet,
826 description: request.description,
827 agent_instructions: request.agent_instructions,
828 version_note: request.version_note,
829 transformation: request.materialize_transformation,
830 strict_metadata: request.strict_metadata,
831 sample_rows: request.sample_rows,
832 vocabulary_threshold: request.vocabulary_threshold,
833 force_metadata_updates: request.force_metadata_updates,
834 variable_registrations: request.variable_registrations,
835 },
836 )
837 .await?;
838
839 Ok(IngestedDatasetFile {
840 source_file,
841 dataset,
842 })
843 }
844
845 pub async fn ingest_file(
846 &self,
847 request: IngestFileRequest,
848 ) -> Result<IngestedFileAsset, AppError> {
849 self.ingest_file_request(None, request).await
850 }
851
852 async fn ingest_file_request(
853 &self,
854 session: Option<&DataStoreSession>,
855 request: IngestFileRequest,
856 ) -> Result<IngestedFileAsset, AppError> {
857 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Ingest
858 {
859 return Err(AppError::Validation(
860 "file ingest requires an ingest transformation".to_string(),
861 ));
862 }
863
864 let source = request.source.clone();
865 let asset_name = request.asset_name.as_str().to_string();
866 let requested_edam_format = request.edam_format.clone();
867 let lake_root = request.lake_root.clone();
868 let validate_without_session = session.is_none();
869 let (content, edam_format) = run_blocking_app_work(move || {
870 acquire_file_workflow_content(
871 &source,
872 &asset_name,
873 requested_edam_format.as_deref(),
874 &lake_root,
875 validate_without_session,
876 )
877 })
878 .await?;
879 let study = self
880 .studies
881 .get_study_by_id(request.study_id)
882 .await
883 .map_err(|error| core_error("lookup ingest study", error))?
884 .ok_or_else(|| {
885 AppError::Validation(format!("ingest study not found: {}", request.study_id.0))
886 })?;
887
888 self.ingest_file_content(
889 session,
890 FileIngestWorkflowRequest {
891 classification: request.classification.clone(),
892 study_id: study.study_id,
893 asset_id: request.asset_id,
894 asset_name: request.asset_name,
895 version_id: request.version_id,
896 content,
897 lake_root: request.lake_root,
898 validate_as_csv: edam_format.eq_ignore_ascii_case("EDAM:format_3752"),
899 edam_format,
900 compress: request.compress,
901 encrypt: request.encrypt,
902 description: request.description,
903 agent_instructions: request.agent_instructions,
904 version_note: request.version_note,
905 transformation: request.transformation,
906 },
907 )
908 .await
909 }
910
911 pub async fn ingest_datafile(
913 &self,
914 session: &DataStoreSession,
915 request: IngestDataFileRequest,
916 ) -> Result<crate::DataFileIngestResult, AppError> {
917 if request.new_version || request.bump_major || request.bump_minor {
918 return Err(AppError::Validation(
919 "Datafile version selection is not supported by this ingest workflow".to_string(),
920 ));
921 }
922 let preflight_deadline = request
923 .content
924 .upload_validation
925 .as_ref()
926 .and_then(|validation| {
927 session.dataset_admission_repository().map(|repository| {
928 repository.upload_deadline(validation.deadline, validation.cancelled.clone())
929 })
930 })
931 .transpose()
932 .map_err(|e| core_error("bound upload preflight", e))?;
933 let study = self
934 .resolve_study_selector(request.study, "ingest Study")
935 .await?;
936 let normalized_format = request.format.trim().to_ascii_lowercase();
937 let validate_as_csv = matches!(
938 normalized_format.as_str(),
939 "csv" | "text/csv" | "edam:format_3752"
940 );
941 let edam_format = match normalized_format.as_str() {
942 "csv" | "text/csv" => "EDAM:format_3752".to_string(),
943 "json" | "application/json" => "EDAM:format_3464".to_string(),
944 _ => resolve_ingest_edam_format(
945 Path::new(request.content.source_file_name()),
946 Some(&request.format),
947 )?,
948 };
949 let upload_provenance = request
950 .content
951 .upload_validation
952 .as_ref()
953 .map(|validation| {
954 format!(
955 "; {} ordinary ingest; byte/row/time budgets: {}",
956 match request.content.acquisition {
957 Some(ahri_tre_types::AcquisitionPath::Trusted) =>
958 "external source Trusted-observed",
959 Some(ahri_tre_types::AcquisitionPath::Client) =>
960 "external source client-asserted",
961 None => "client-uploaded",
962 },
963 serde_json::to_string(&validation.budgets).expect("finite budgets serialize")
964 )
965 })
966 .unwrap_or_default();
967 let description = request.description;
968 let asset_label = request.asset_name.as_str().to_string();
969 drop(preflight_deadline);
970 let ingested = self
971 .ingest_file_content(
972 Some(session),
973 FileIngestWorkflowRequest {
974 classification: request.classification.clone(),
975 study_id: study.study_id,
976 asset_id: None,
977 asset_name: request.asset_name,
978 version_id: None,
979 content: request.content,
980 lake_root: session.runtime().lake.data_path.clone(),
981 validate_as_csv,
982 edam_format,
983 compress: request.compress,
984 encrypt: Some(crate::resolve_file_asset_encryption(request.encrypt)),
985 description: description.clone(),
986 agent_instructions: None,
987 version_note: description,
988 transformation: ahri_tre_types::NewTransformationRecord {
989 transformation_type: ahri_tre_types::TransformationType::Ingest,
990 description: format!("ingest datafile {asset_label}{upload_provenance}"),
991 repository_url: None,
992 commit_hash: None,
993 file_path: None,
994 date_created: None,
995 created_by: None,
996 },
997 },
998 )
999 .await?;
1000 Ok(crate::DataFileIngestResult::from_governed_result(
1001 study, ingested,
1002 ))
1003 }
1004
1005 async fn ingest_file_content(
1006 &self,
1007 session: Option<&DataStoreSession>,
1008 request: FileIngestWorkflowRequest,
1009 ) -> Result<IngestedFileAsset, AppError> {
1010 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Ingest
1011 {
1012 return Err(AppError::Validation(
1013 "file ingest requires an ingest transformation".to_string(),
1014 ));
1015 }
1016 if request.validate_as_csv
1017 && request
1018 .content
1019 .media_type()
1020 .is_some_and(|media_type| !media_type.eq_ignore_ascii_case("text/csv"))
1021 {
1022 return Err(AppError::Validation(
1023 "CSV Datafile content has an invalid media type".to_string(),
1024 ));
1025 }
1026 let preflight_deadline = request
1027 .content
1028 .upload_validation
1029 .as_ref()
1030 .and_then(|validation| {
1031 session
1032 .and_then(DataStoreSession::dataset_admission_repository)
1033 .map(|repository| {
1034 repository
1035 .upload_deadline(validation.deadline, validation.cancelled.clone())
1036 })
1037 })
1038 .transpose()
1039 .map_err(|e| core_error("bound upload preflight", e))?;
1040 let workflow_guard = if let Some(validation) = &request.content.upload_validation {
1041 validation
1042 .check_deadline()
1043 .map_err(|_| AppError::Validation("Upload expired or cancelled".into()))?;
1044 Arc::clone(&FILE_INGEST_WORKFLOW_LOCK)
1045 .try_lock_owned()
1046 .map_err(|_| AppError::Conflict("File ingest is busy".into()))?
1047 } else {
1048 Arc::clone(&FILE_INGEST_WORKFLOW_LOCK).lock_owned().await
1049 };
1050
1051 let existing_asset = self
1052 .assets
1053 .get_asset_by_name(request.study_id, request.asset_name.as_str())
1054 .await
1055 .map_err(|error| core_error("lookup ingest asset", error))?;
1056 let asset_extraction = ahri_tre_core::extract_inline_tags(request.description.as_deref());
1057 let version_extraction =
1058 ahri_tre_core::extract_inline_tags(request.version_note.as_deref());
1059
1060 let (asset, asset_persisted) = match existing_asset {
1061 Some(asset) => {
1062 if let Some(requested_asset_id) = request.asset_id
1063 && requested_asset_id != asset.asset_id
1064 {
1065 return Err(AppError::Validation(format!(
1066 "ingest asset id {} does not match existing asset {} named {}",
1067 requested_asset_id.0,
1068 asset.asset_id.0,
1069 request.asset_name.as_str()
1070 )));
1071 }
1072 if asset.asset_type != ahri_tre_types::AssetType::File {
1073 return Err(AppError::Validation(format!(
1074 "ingest asset {} is not a file asset",
1075 asset.asset_id.0
1076 )));
1077 }
1078 (asset, true)
1079 }
1080 None => {
1081 let new_asset = ahri_tre_types::NewAssetRecord {
1082 study_id: request.study_id,
1083 name: request.asset_name.clone(),
1084 description: asset_extraction.cleaned_text.clone(),
1085 agent_instructions: request.agent_instructions.clone(),
1086 asset_type: ahri_tre_types::AssetType::File,
1087 date_created: None,
1088 created_by: request.transformation.created_by.clone(),
1089 };
1090 (
1091 ahri_tre_types::AssetRecord {
1092 asset_id: request
1093 .asset_id
1094 .unwrap_or_else(|| ahri_tre_types::AssetId(uuid::Uuid::new_v4())),
1095 study_id: new_asset.study_id,
1096 name: new_asset.name,
1097 description: new_asset.description,
1098 agent_instructions: new_asset.agent_instructions,
1099 asset_type: new_asset.asset_type,
1100 date_created: new_asset.date_created,
1101 created_by: new_asset.created_by,
1102 },
1103 false,
1104 )
1105 }
1106 };
1107 let tags_before_ingest = self
1108 .tags
1109 .list_tags()
1110 .await
1111 .map_err(|error| core_error("list tags before ingest", error))?;
1112 let asset_tags_before_ingest = if asset_persisted {
1113 self.tags
1114 .list_tags_for_target(TagAttachmentTarget::Asset(asset.asset_id))
1115 .await
1116 .map_err(|error| core_error("list asset tags before ingest", error))?
1117 } else {
1118 Vec::new()
1119 };
1120
1121 let existing_versions = if !asset_persisted {
1122 Vec::new()
1123 } else {
1124 self.assets
1125 .list_asset_versions(asset.asset_id)
1126 .await
1127 .map_err(|error| core_error("list ingest asset versions", error))?
1128 };
1129 let (major, minor, patch) =
1130 ahri_tre_core::next_asset_version_coordinates(&existing_versions)
1131 .map_err(|error| core_error("choose ingest asset version", error))?;
1132 let version = if let Some(version_id) = request.version_id {
1133 let version = ahri_tre_types::AssetVersionRecord {
1134 version_id,
1135 asset_id: asset.asset_id,
1136 major,
1137 minor,
1138 patch,
1139 version_label: None,
1140 version_note: version_extraction.cleaned_text.clone(),
1141 is_latest: None,
1142 doi: None,
1143 created_at: None,
1144 created_by: request.transformation.created_by.clone(),
1145 };
1146 ahri_tre_core::validate_new_asset_version(&existing_versions, &version)
1147 .map_err(|error| core_error("validate ingest asset version", error))?;
1148 version
1149 } else {
1150 let version = ahri_tre_types::NewAssetVersionRecord {
1151 classification: request.classification.clone(),
1152 asset_id: asset.asset_id,
1153 major,
1154 minor,
1155 patch,
1156 version_label: None,
1157 version_note: version_extraction.cleaned_text.clone(),
1158 is_latest: None,
1159 doi: None,
1160 created_at: None,
1161 created_by: request.transformation.created_by.clone(),
1162 };
1163 ahri_tre_core::validate_new_asset_version_input(&existing_versions, &version)
1164 .map_err(|error| core_error("validate ingest asset version", error))?;
1165 let version = ahri_tre_core::prepare_new_asset_version_for_create(version)
1166 .map_err(|error| core_error("prepare ingest asset version", error))?;
1167 ahri_tre_types::AssetVersionRecord {
1168 version_id: ahri_tre_types::VersionId(uuid::Uuid::new_v4()),
1169 asset_id: version.asset_id,
1170 major: version.major,
1171 minor: version.minor,
1172 patch: version.patch,
1173 version_label: version.version_label,
1174 version_note: version.version_note,
1175 is_latest: version.is_latest,
1176 doi: version.doi,
1177 created_at: version.created_at,
1178 created_by: version.created_by,
1179 }
1180 };
1181 let version_persisted = false;
1182
1183 let lake = DuckLakeAdapter::new(&request.lake_root);
1184 let validate_as_csv = request.validate_as_csv;
1185 let declared_content_length = request.content.content_length();
1186 let declared_digest = request.content.declared_digest().map(str::to_string);
1187 let stage_request = StageDataFileStreamRequest {
1188 source_file_name: request.content.source_file_name().to_string(),
1189 asset_id: asset.asset_id,
1190 version_id: version.version_id,
1191 study_id: request.study_id,
1192 asset_name: asset.name.clone(),
1193 major: version.major,
1194 minor: version.minor,
1195 patch: version.patch,
1196 edam_format: request.edam_format,
1197 compression: if request.compress {
1198 DataFileCompression::Zstd
1199 } else {
1200 DataFileCompression::None
1201 },
1202 encryption: if request.encrypt.unwrap_or(true) {
1203 DataFileEncryption::Aes256Cbc
1204 } else {
1205 DataFileEncryption::None
1206 },
1207 };
1208 let validation =
1209 validate_as_csv.then(|| (DatasetFileFormat::Csv, DatasetFileReadOptions::default()));
1210 let scratch = match session {
1211 Some(session) => Some(session.shared_scratch_attempt().ok_or_else(|| {
1212 AppError::Infrastructure(
1213 "trusted scratch is not available for Datafile ingest".to_string(),
1214 )
1215 })?),
1216 None => None,
1217 };
1218 let upload_validation = request.content.upload_validation.clone();
1219 drop(preflight_deadline);
1220 let stage_validation = upload_validation.clone();
1221 let mut reader = request.content.into_reader();
1222 let stage_lake = lake.clone();
1223 let mut staged_guard = run_blocking_lake_work(move || {
1224 let staged = match scratch {
1225 Some(scratch) => stage_lake.stage_datafile_content_validated(
1226 &scratch,
1227 &stage_request,
1228 validation,
1229 stage_validation.as_ref(),
1230 reader.as_mut(),
1231 ),
1232 None => stage_lake.stage_datafile_stream(&stage_request, reader.as_mut()),
1233 }?;
1234 Ok(StagedFileGuard {
1235 staged: Some(staged),
1236 lake: stage_lake,
1237 workflow_guard: Some(workflow_guard),
1238 })
1239 })
1240 .await
1241 .map_err(|error| match error {
1242 ahri_tre_lake::LakeError::DuckDb(_) if validate_as_csv => {
1243 AppError::Validation("Datafile content is invalid".to_string())
1244 }
1245 ahri_tre_lake::LakeError::DataFileAlreadyExists { .. } => {
1246 AppError::Conflict("Datafile payload already exists".to_string())
1247 }
1248 other => infrastructure_error("stage ingest datafile", other),
1249 })?;
1250 let staged = staged_guard.staged();
1251
1252 if let Some(validation) = &upload_validation {
1253 validation
1254 .check_deadline()
1255 .map_err(|e| infrastructure_error("upload deadline", e))?;
1256 }
1257 let declared_length_matches =
1258 declared_content_length.is_none_or(|length| length == staged.source_size_bytes);
1259 let declared_digest_matches = declared_digest.as_deref().is_none_or(|digest| {
1260 digest.strip_prefix("sha256:").unwrap_or(digest) == staged.source_digest.as_str()
1261 });
1262 if !declared_length_matches || !declared_digest_matches {
1263 let rollback_lake = lake.clone();
1264 let rollback_datafile = staged.datafile.clone();
1265 let physical_rollback = run_blocking_lake_work(move || {
1266 rollback_lake.delete_datafile(&DeleteDataFileRequest {
1267 datafile: rollback_datafile,
1268 })
1269 })
1270 .await;
1271 if physical_rollback.is_ok() {
1272 staged_guard.disarm();
1273 }
1274 physical_rollback
1275 .map_err(|error| infrastructure_error("rollback invalid ingest content", error))?;
1276 return Err(AppError::Validation(
1277 "Datafile content does not match its declared facts".to_string(),
1278 ));
1279 }
1280
1281 let file_operator = session.and_then(DataStoreSession::governance_maintenance);
1282 let file_root = request.lake_root.clone();
1283 let metadata_service = self.clone();
1284 let staged_datafile = staged_guard.staged().datafile.clone();
1285 let requested_asset_tags = asset_extraction.tags;
1286 let requested_version_tags = version_extraction.tags;
1287 let classification = request.classification.clone();
1288 let transformation = request.transformation;
1289 let upload_repository = session.and_then(DataStoreSession::dataset_admission_repository);
1290 let (catalog, datafile, lineage, staged) = run_cancellation_safe_app_work(async move {
1291 let metadata_result = async {
1292 let _deadline = match (&upload_validation, upload_repository) {
1293 (Some(validation), Some(repository)) => Some(repository.upload_deadline(validation.deadline, validation.cancelled.clone()).map_err(|e| core_error("bound upload metadata", e))?),
1294 _ => None,
1295 };
1296 Self::bind_staged_datafile(file_operator, file_root, asset.study_id, asset.asset_id, staged_datafile.clone()).await?;
1297 if !asset_persisted {
1298 metadata_service
1299 .assets
1300 .save_asset(&asset)
1301 .await
1302 .map_err(|error| core_error("save ingest asset", error))?;
1303 }
1304 metadata_service
1305 .attach_tags(
1306 TagAttachmentTarget::Asset(asset.asset_id),
1307 requested_asset_tags.clone(),
1308 )
1309 .await?;
1310
1311 if !version_persisted {
1312 metadata_service
1313 .assets
1314 .admit_asset_version(&version,&classification)
1315 .await
1316 .map_err(|error| core_error("save ingest asset version", error))?;
1317 }
1318 metadata_service
1319 .attach_tags(
1320 TagAttachmentTarget::AssetVersion(version.version_id),
1321 requested_version_tags.clone(),
1322 )
1323 .await?;
1324 let datafile = metadata_service
1325 .assets
1326 .save_datafile(&staged_datafile)
1327 .await
1328 .map_err(|error| core_error("save ingest datafile", error))?;
1329 let policy = DefaultProvenancePolicy;
1330 let transformation = enrich_transformation_source(&transformation);
1331 let lineage = ahri_tre_core::record_ingest_provenance(
1332 metadata_service.transformations.as_ref(),
1333 &policy,
1334 &transformation,
1335 &[version.version_id],
1336 )
1337 .await
1338 .map_err(|error| core_error("record ingest provenance", error))?;
1339 let catalog = metadata_service
1340 .get_asset(GetAssetRequest {
1341 asset_id: Some(asset.asset_id),
1342 study_id: None,
1343 asset_name: None,
1344 })
1345 .await?;
1346
1347 if let Some(validation) = &upload_validation {
1348 validation.check_deadline().map_err(|e| infrastructure_error("upload admission deadline", e))?;
1349 }
1350 Ok::<_, AppError>((catalog, datafile, lineage))
1351 }
1352 .await;
1353
1354 let (catalog, datafile, lineage) = match metadata_result {
1355 Ok(records) => records,
1356 Err(error) => {
1357 let rollback_lake = lake.clone();
1358 let rollback_datafile = staged_datafile;
1359 let physical_rollback = run_blocking_lake_work(move || {
1360 rollback_lake.delete_datafile(&DeleteDataFileRequest {
1361 datafile: rollback_datafile,
1362 })
1363 })
1364 .await;
1365 let rollback = metadata_service
1366 .rollback_failed_file_ingest(FileIngestRollbackContext {
1367 asset_id: asset.asset_id,
1368 asset_preexisted: asset_persisted,
1369 version_id: version.version_id,
1370 tags_before_ingest: &tags_before_ingest,
1371 asset_tags_before_ingest: &asset_tags_before_ingest,
1372 requested_asset_tags: &requested_asset_tags,
1373 requested_version_tags: &requested_version_tags,
1374 })
1375 .await;
1376 if physical_rollback.is_ok() {
1377 staged_guard.disarm();
1378 }
1379 return match (physical_rollback, rollback) {
1380 (Ok(_), Ok(())) => Err(error),
1381 (physical, metadata) => Err(AppError::Infrastructure(format!(
1382 "file ingest failed and rollback also failed: ingest={error}; physical={physical:?}; metadata={metadata:?}"
1383 ))),
1384 };
1385 }
1386 };
1387
1388 let staged = staged_guard.commit();
1389 Ok((catalog, datafile, lineage, staged))
1390 })
1391 .await?;
1392
1393 Ok(IngestedFileAsset {
1394 catalog,
1395 datafile,
1396 lineage,
1397 source_size_bytes: staged.source_size_bytes,
1398 stored_size_bytes: staged.staged_size_bytes,
1399 })
1400 }
1401
1402 async fn rollback_failed_file_ingest(
1403 &self,
1404 context: FileIngestRollbackContext<'_>,
1405 ) -> Result<(), AppError> {
1406 let transformations = self
1407 .transformations
1408 .list_transformations_producing(context.version_id)
1409 .await
1410 .map_err(|error| core_error("find ingest provenance for rollback", error))?;
1411 for transformation in transformations {
1412 let deleted = self
1413 .transformations
1414 .delete_transformation(transformation.transformation_id)
1415 .await
1416 .map_err(|error| core_error("rollback ingest provenance", error))?;
1417 if !deleted {
1418 return Err(AppError::Infrastructure(
1419 "rollback did not remove ingest provenance".to_string(),
1420 ));
1421 }
1422 }
1423 let current_asset_tags = self
1424 .tags
1425 .list_tags_for_target(TagAttachmentTarget::Asset(context.asset_id))
1426 .await
1427 .map_err(|error| core_error("list ingest asset tags for rollback", error))?;
1428 for tag in current_asset_tags {
1429 if context
1430 .requested_asset_tags
1431 .iter()
1432 .any(|name| name == &tag.name)
1433 && (!context.asset_preexisted
1434 || !context
1435 .asset_tags_before_ingest
1436 .iter()
1437 .any(|existing| existing.tag_id == tag.tag_id))
1438 {
1439 self.tags
1440 .detach_tag(tag.tag_id, TagAttachmentTarget::Asset(context.asset_id))
1441 .await
1442 .map_err(|error| core_error("rollback ingest asset tag", error))?;
1443 }
1444 }
1445 let current_version_tags = self
1446 .tags
1447 .list_tags_for_target(TagAttachmentTarget::AssetVersion(context.version_id))
1448 .await
1449 .map_err(|error| core_error("list ingest version tags for rollback", error))?;
1450 for tag in current_version_tags {
1451 if context
1452 .requested_version_tags
1453 .iter()
1454 .any(|name| name == &tag.name)
1455 {
1456 self.tags
1457 .detach_tag(
1458 tag.tag_id,
1459 TagAttachmentTarget::AssetVersion(context.version_id),
1460 )
1461 .await
1462 .map_err(|error| core_error("rollback ingest version tag", error))?;
1463 }
1464 }
1465 self.assets
1466 .delete_datafile_record(context.version_id)
1467 .await
1468 .map_err(|error| core_error("rollback ingest datafile", error))?;
1469 self.assets
1470 .delete_asset_version_record(context.version_id)
1471 .await
1472 .map_err(|error| core_error("rollback ingest asset version", error))?;
1473 if !context.asset_preexisted {
1474 self.assets
1475 .delete_asset_record(context.asset_id)
1476 .await
1477 .map_err(|error| core_error("rollback ingest asset", error))?;
1478 }
1479 let tags_after_rollback = self
1480 .tags
1481 .list_tags()
1482 .await
1483 .map_err(|error| core_error("list ingest tags for rollback", error))?;
1484 for tag in tags_after_rollback {
1485 if context
1486 .requested_asset_tags
1487 .iter()
1488 .chain(context.requested_version_tags)
1489 .any(|name| name == &tag.name)
1490 && !context
1491 .tags_before_ingest
1492 .iter()
1493 .any(|existing| existing.tag_id == tag.tag_id)
1494 {
1495 self.tags
1496 .delete_tag_if_unused(tag.tag_id)
1497 .await
1498 .map_err(|error| core_error("rollback unused ingest tag", error))?;
1499 }
1500 }
1501 Ok(())
1502 }
1503
1504 pub async fn ingest_file_for_session(
1505 &self,
1506 session: &DataStoreSession,
1507 mut request: IngestFileRequest,
1508 ) -> Result<IngestedFileAsset, AppError> {
1509 request.lake_root = session.runtime().lake.data_path.clone();
1510 request.encrypt = Some(crate::resolve_file_asset_encryption(request.encrypt));
1511 self.ingest_file_request(Some(session), request).await
1512 }
1513
1514 pub async fn ingest_redcap_offline(
1515 &self,
1516 session: &mut DataStoreSession,
1517 request: OfflineRedcapIngestRequest,
1518 ) -> Result<RedcapIngestResult, AppError> {
1519 self.ingest_redcap_offline_bounded(session, request, None)
1520 .await
1521 }
1522
1523 async fn ingest_redcap_offline_bounded(
1524 &self,
1525 session: &mut DataStoreSession,
1526 request: OfflineRedcapIngestRequest,
1527 limits: Option<&ahri_tre_lake::UploadValidation>,
1528 ) -> Result<RedcapIngestResult, AppError> {
1529 let metadata_deadline = limits
1532 .and_then(|limits| {
1533 session.dataset_admission_repository().map(|repository| {
1534 repository.upload_deadline(limits.deadline, limits.cancelled.clone())
1535 })
1536 })
1537 .transpose()
1538 .map_err(|error| core_error("bound REDCap artifact publication", error))?;
1539 if request.ingest_transformation.transformation_type
1540 != ahri_tre_types::TransformationType::Ingest
1541 {
1542 return Err(AppError::Validation(
1543 "offline REDCap ingest requires an ingest transformation".to_string(),
1544 ));
1545 }
1546 if request.transform_transformation.transformation_type
1547 != ahri_tre_types::TransformationType::Transform
1548 {
1549 return Err(AppError::Validation(
1550 "offline REDCap dataset materialization requires a transform transformation"
1551 .to_string(),
1552 ));
1553 }
1554
1555 let project_info = parse_project_info_file(&request.project_info_path)
1556 .map_err(|error| redcap_error("parse REDCap project info", error))?;
1557 let dictionary = RedcapDictionary::from_file(&request.metadata_path)
1558 .map_err(|error| redcap_error("parse REDCap metadata", error))?;
1559 if dictionary
1560 .forms()
1561 .iter()
1562 .map(|form| form.fields.len())
1563 .sum::<usize>()
1564 > 4096
1565 {
1566 return Err(AppError::Validation(
1567 "REDCap dictionary exceeds field limit".into(),
1568 ));
1569 }
1570 let mut forms = dictionary
1571 .selected_forms(&request.forms)
1572 .map_err(|error| redcap_error("select REDCap forms", error))?;
1573 apply_redcap_skip_forms(&mut forms, &request.skip_forms);
1574 if forms.is_empty() {
1575 return Err(AppError::Validation(redcap_empty_form_selection_message(
1576 &request.skip_forms,
1577 )));
1578 }
1579
1580 if forms.len() > 256 {
1581 return Err(AppError::Validation(
1582 "REDCap selection exceeds form limit".into(),
1583 ));
1584 }
1585 let mut names = std::collections::BTreeSet::new();
1586 for form in &forms {
1587 if form.fields.len() > 1024
1588 || !names.insert(
1589 redcap_dataset_name(&project_info, form, request.dataset_prefix.as_ref())?
1590 .as_str()
1591 .to_string(),
1592 )
1593 {
1594 return Err(AppError::Validation(
1595 "REDCap form columns or output names are invalid".into(),
1596 ));
1597 }
1598 }
1599 let domain = self
1600 .ensure_redcap_domain(
1601 &project_info,
1602 request.domain_id,
1603 request.domain_name.as_ref(),
1604 )
1605 .await?;
1606 let study = self
1607 .resolve_redcap_study(
1608 &project_info,
1609 RedcapStudyResolveContext {
1610 study_id: request.study_id,
1611 domain_id: domain.domain_id,
1612 study_name: request.study_name.as_ref(),
1613 agent_instructions: request.agent_instructions.as_ref(),
1614 ingest_transformation: &request.ingest_transformation,
1615 },
1616 )
1617 .await?;
1618 self.ensure_study_domain(study.study_id, domain.domain_id, "REDCap ingest")
1619 .await?;
1620
1621 let lake_root = session.runtime().lake.data_path.clone();
1622 let encrypt = crate::resolve_file_asset_encryption(request.encrypt);
1623 let artifact_prefix = redcap_asset_prefix(&project_info)?;
1624 let artifacts = vec![
1625 RedcapArtifactInput {
1626 suffix: "project_info",
1627 path: PathBuf::from(&request.project_info_path),
1628 edam_format: "EDAM:format_3464",
1629 description: format!(
1630 "REDCap project {} project info for {}",
1631 project_info.project_id, project_info.project_title
1632 ),
1633 },
1634 RedcapArtifactInput {
1635 suffix: "metadata",
1636 path: PathBuf::from(&request.metadata_path),
1637 edam_format: "EDAM:format_3464",
1638 description: format!(
1639 "REDCap project {} metadata for {}",
1640 project_info.project_id, project_info.project_title
1641 ),
1642 },
1643 RedcapArtifactInput {
1644 suffix: "records_eav",
1645 path: PathBuf::from(&request.records_eav_path),
1646 edam_format: "EDAM:format_3752",
1647 description: format!(
1648 "REDCap project {} EAV records for {}",
1649 project_info.project_id, project_info.project_title
1650 ),
1651 },
1652 ];
1653
1654 let mut staged_artifacts = Vec::new();
1655 for artifact in artifacts {
1656 staged_artifacts.push(
1657 self.stage_redcap_artifact(
1658 RedcapArtifactStageContext {
1659 limits,
1660 operator: session.governance_maintenance(),
1661 classification: &request.classification,
1662 study_id: study.study_id,
1663 lake_root: &lake_root,
1664 asset_prefix: &artifact_prefix,
1665 compress: request.compress,
1666 encrypt,
1667 agent_instructions: request.agent_instructions.as_ref(),
1668 version_note: request.version_note.as_ref(),
1669 created_by: request.ingest_transformation.created_by.as_ref(),
1670 },
1671 artifact,
1672 )
1673 .await?,
1674 );
1675 }
1676 let artifact_versions = staged_artifacts
1677 .iter()
1678 .map(|artifact| artifact.version.version_id)
1679 .collect::<Vec<_>>();
1680 let policy = DefaultProvenancePolicy;
1681 let ingest_transformation = enrich_transformation_source(&request.ingest_transformation);
1682 let ingest_lineage = ahri_tre_core::record_ingest_provenance(
1683 self.transformations.as_ref(),
1684 &policy,
1685 &ingest_transformation,
1686 &artifact_versions,
1687 )
1688 .await
1689 .map_err(|error| core_error("record REDCap artifact ingest provenance", error))?;
1690
1691 let mut datafiles = Vec::new();
1692 for artifact in &staged_artifacts {
1693 datafiles.push(IngestedFileAsset {
1694 catalog: self
1695 .get_asset(GetAssetRequest {
1696 asset_id: Some(artifact.asset.asset_id),
1697 study_id: None,
1698 asset_name: None,
1699 })
1700 .await?,
1701 datafile: artifact.datafile.clone(),
1702 lineage: ingest_lineage.clone(),
1703 source_size_bytes: artifact.source_size_bytes,
1704 stored_size_bytes: artifact.stored_size_bytes,
1705 });
1706 }
1707
1708 let eav_datafile = staged_artifacts
1709 .iter()
1710 .find(|artifact| artifact.suffix == "records_eav")
1711 .ok_or_else(|| {
1712 AppError::Infrastructure("staged REDCap EAV artifact missing".to_string())
1713 })?;
1714 let lake = DuckLakeAdapter::new(&lake_root);
1715
1716 drop(metadata_deadline);
1717 let result = self
1718 .materialize_redcap_forms(
1719 session,
1720 &lake,
1721 &eav_datafile.datafile,
1722 &study,
1723 domain.domain_id,
1724 &project_info,
1725 &forms,
1726 &artifact_versions,
1727 RedcapMaterializeOptions {
1728 limits,
1729 no_register_dictionary: request.no_register_dictionary,
1730 dataset_prefix: request.dataset_prefix.as_ref(),
1731 agent_instructions: request.agent_instructions.as_ref(),
1732 version_note: request.version_note.as_ref(),
1733 transform_transformation: &request.transform_transformation,
1734 force_metadata_updates: request.force_metadata_updates,
1735 },
1736 )
1737 .await;
1738 let (datasets, warnings, form_outcomes) = result?;
1739
1740 Ok(RedcapIngestResult {
1741 form_outcomes,
1742 study,
1743 datafiles,
1744 ingest_lineage,
1745 datasets,
1746 warnings,
1747 })
1748 }
1749
1750 pub async fn open_duckdb_source(
1751 &self,
1752 request: OpenDuckDbSourceRequest,
1753 ) -> Result<SqlSourceConnection, AppError> {
1754 SqlSourceConnection::open_duckdb(request)
1755 }
1756
1757 pub async fn open_sqlite_source(
1758 &self,
1759 request: OpenSqliteSourceRequest,
1760 ) -> Result<SqlSourceConnection, AppError> {
1761 SqlSourceConnection::open_sqlite(request)
1762 }
1763
1764 pub async fn open_postgres_source(
1765 &self,
1766 request: OpenPostgresSourceRequest,
1767 ) -> Result<SqlSourceConnection, AppError> {
1768 SqlSourceConnection::open_postgres(request)
1769 }
1770
1771 pub async fn open_mssql_source(
1772 &self,
1773 request: OpenMsSqlSourceRequest,
1774 ) -> Result<SqlSourceConnection, AppError> {
1775 SqlSourceConnection::open_mssql(request)
1776 }
1777
1778 pub async fn analyze_sql_source_metadata(
1779 &self,
1780 source: &mut SqlSourceConnection,
1781 request: AnalyzeSqlSourceRequest,
1782 ) -> Result<SqlSourceAnalysis, AppError> {
1783 source.analyze(request)
1784 }
1785
1786 pub async fn sql_to_dataset(
1787 &self,
1788 session: &mut DataStoreSession,
1789 source: &mut SqlSourceConnection,
1790 request: SqlToDatasetRequest,
1791 ) -> Result<DatasetMaterialization, AppError> {
1792 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Ingest
1793 {
1794 return Err(AppError::Validation(
1795 "sql-to-dataset requires an ingest transformation".to_string(),
1796 ));
1797 }
1798
1799 let study_domains = self
1800 .study_domains
1801 .list_domains_for_study(request.study_id)
1802 .await
1803 .map_err(|error| core_error("list study domains for SQL ingest", error))?;
1804 if !study_domains
1805 .iter()
1806 .any(|domain| domain.domain_id == request.domain_id)
1807 {
1808 return Err(AppError::Validation(format!(
1809 "SQL ingest domain {} is not linked to study {}",
1810 request.domain_id.0, request.study_id.0
1811 )));
1812 }
1813
1814 let analysis = source.analyze(AnalyzeSqlSourceRequest {
1815 sql: request.sql.clone(),
1816 })?;
1817 let prepared_query = source.prepare_lake_query(session.lake_connection(), &request.sql)?;
1818 let variable_registrations =
1819 self.sql_analysis_variable_registrations(request.domain_id, &analysis)?;
1820
1821 let mut attempt = self
1822 .prepare_dataset_attempt(
1823 session,
1824 PrepareDatasetVersionRequest {
1825 output_intent: None,
1826 inherit_risk: false,
1827 classification: request.classification.clone(),
1828 study_id: request.study_id,
1829 dataset_asset_id: request.dataset_asset_id,
1830 dataset_name: request.dataset_name,
1831 dataset_version_id: request.dataset_version_id,
1832 description: request.description,
1833 agent_instructions: request.agent_instructions,
1834 version_note: request.version_note,
1835 created_by: request.transformation.created_by.clone(),
1836 },
1837 )
1838 .await?;
1839 let prepared = attempt.prepared.clone();
1840 let result = async {
1841 let lake_root = session.runtime().lake.data_path.clone();
1842 let lake = DuckLakeAdapter::new(&lake_root);
1843 let loaded = lake
1844 .load_dataset_table_from_sql_reserved(
1845 session.lake_connection(),
1846 &mut attempt.authority,
1847 &LoadDatasetTableFromSqlRequest {
1848 study_id: request.study_id,
1849 dataset_name: prepared.asset.name.clone(),
1850 major: prepared.version.major,
1851 minor: prepared.version.minor,
1852 patch: prepared.version.patch,
1853 source_sql: prepared_query.sql,
1854 },
1855 )
1856 .map_err(|error| infrastructure_error("load SQL source into DuckLake", error))?;
1857 let relation = loaded.relation.clone();
1858 let dataset_record = ahri_tre_types::DatasetRecord {
1859 dataset_id: prepared.version.version_id,
1860 };
1861
1862 let transformation = enrich_transformation_source(&request.transformation);
1863 let metadata_result = async {
1864 let (catalog, dataset, lineage, variables) = self
1865 .save_dataset_metadata_with_options(
1866 session,
1867 &mut attempt,
1868 dataset_record,
1869 &transformation,
1870 &[],
1871 DatasetMetadataRegistration {
1872 redcap_dictionary: None,
1873 variable_registrations,
1874 loaded_column_names: Some(&loaded.column_names),
1875 force_metadata_updates: request.force_metadata_updates,
1876 },
1877 )
1878 .await?;
1879 Ok::<_, AppError>((catalog, dataset, lineage, variables))
1880 }
1881 .await;
1882
1883 let (catalog, dataset, lineage, variables) = metadata_result?;
1884
1885 Ok(DatasetMaterialization {
1886 catalog,
1887 dataset,
1888 lineage,
1889 lake_relation: relation.qualified_name(),
1890 lake_schema: relation.schema_name,
1891 lake_table: relation.table_name,
1892 row_count: loaded.row_count,
1893 column_count: loaded.column_count,
1894 variables,
1895 metadata_warnings: Vec::new(),
1896 })
1897 }
1898 .await;
1899 attempt.finish(session, result)
1900 }
1901
1902 pub async fn transform_dataset_with_lake_sql(
1903 &self,
1904 session: &mut DataStoreSession,
1905 request: DatasetSqlTransformRequest,
1906 ) -> Result<DatasetMaterialization, AppError> {
1907 if request.transformation.transformation_type
1908 != ahri_tre_types::TransformationType::Transform
1909 {
1910 return Err(AppError::Validation(
1911 "dataset SQL transform requires a transform transformation".to_string(),
1912 ));
1913 }
1914
1915 let mut bindings = request
1916 .inputs
1917 .iter()
1918 .map(|(alias, version)| {
1919 (
1920 alias.clone(),
1921 ahri_tre_lake::QueryBinding::Dataset(version.0),
1922 )
1923 })
1924 .collect::<std::collections::BTreeMap<_, _>>();
1925 for (alias, sql) in &request.views {
1926 if bindings
1927 .insert(
1928 alias.clone(),
1929 ahri_tre_lake::QueryBinding::View(sql.clone()),
1930 )
1931 .is_some()
1932 {
1933 return Err(AppError::Conflict("Duplicate transform binding".into()));
1934 }
1935 }
1936 let plan =
1937 ahri_tre_lake::analyze_disclosure_query(&request.sql, &bindings).map_err(|_| {
1938 AppError::Validation("SQL transform dependencies are not admissible".into())
1939 })?;
1940 if plan.input_versions().is_empty() {
1941 return Err(AppError::Validation(
1942 "SQL transform requires governed inputs".into(),
1943 ));
1944 }
1945 let inputs = plan
1946 .input_versions()
1947 .iter()
1948 .copied()
1949 .map(ahri_tre_types::VersionId)
1950 .collect::<Vec<_>>();
1951 let budgets = session
1952 .runtime()
1953 .disclosure
1954 .resolve(&request.budgets)
1955 .ok_or_else(|| {
1956 AppError::Validation(
1957 "Transform budgets exceed configured ceilings or are invalid".into(),
1958 )
1959 })?;
1960 let deadline = Utc::now() + chrono::Duration::seconds(i64::from(budgets.total_seconds));
1961 if session
1962 .disclosure_context
1963 .as_ref()
1964 .is_some_and(|(_, auth)| {
1965 auth.runtime_expires_at()
1966 .is_some_and(|expiry| expiry < deadline)
1967 })
1968 {
1969 return Err(AppError::Validation(
1970 "Transform budget exceeds this Session's remaining lifetime".into(),
1971 ));
1972 }
1973 let deadline_instant = std::time::Instant::now()
1974 + std::time::Duration::from_secs(u64::from(budgets.total_seconds));
1975 let cancelled = Arc::new(std::sync::atomic::AtomicBool::new(false));
1976 let metadata_deadline = session
1977 .dataset_admission_repository()
1978 .map(|repository| repository.upload_deadline(deadline_instant, cancelled.clone()))
1979 .transpose()
1980 .map_err(|error| core_error("bound transform metadata", error))?;
1981 let monitor = upload::UploadMonitor::new(session, deadline_instant, cancelled);
1982 let mut attempt = self
1983 .prepare_dataset_attempt(
1984 session,
1985 PrepareDatasetVersionRequest {
1986 output_intent: None,
1987 inherit_risk: false,
1988 classification: ahri_tre_types::IngestClassification::derived_high(),
1989 study_id: request.study_id,
1990 dataset_asset_id: request.dataset_asset_id,
1991 dataset_name: request.dataset_name,
1992 dataset_version_id: request.dataset_version_id,
1993 description: request.description,
1994 agent_instructions: request.agent_instructions,
1995 version_note: request.version_note,
1996 created_by: request.transformation.created_by.clone(),
1997 },
1998 )
1999 .await?;
2000 let prepared = attempt.prepared.clone();
2001 let result = async {
2002 let output = Self::prepare_dataset_sql_transform(
2003 session,
2004 &attempt.scratch,
2005 plan,
2006 budgets,
2007 deadline,
2008 )
2009 .await?;
2010 let lake_root = session.runtime().lake.data_path.clone();
2011 let lake = DuckLakeAdapter::new(&lake_root);
2012 let loaded = lake
2013 .load_restricted_query_output_reserved(
2014 session.lake_connection(),
2015 &mut attempt.authority,
2016 &LoadDatasetTableFromSqlRequest {
2017 study_id: request.study_id,
2018 dataset_name: prepared.asset.name.clone(),
2019 major: prepared.version.major,
2020 minor: prepared.version.minor,
2021 patch: prepared.version.patch,
2022 source_sql: String::new(),
2023 },
2024 output,
2025 )
2026 .map_err(|error| {
2027 infrastructure_error("load dataset SQL transform into DuckLake", error)
2028 })?;
2029 let relation = loaded.relation.clone();
2030 let dataset_record = ahri_tre_types::DatasetRecord {
2031 dataset_id: prepared.version.version_id,
2032 };
2033
2034 if Utc::now() >= deadline {
2035 return Err(AppError::Validation("Transform deadline elapsed".into()));
2036 }
2037 let transformation = enrich_transformation_source(&request.transformation);
2038 let metadata_result = async {
2039 let (catalog, dataset, lineage, variables) = self
2040 .save_dataset_metadata_with_options(
2041 session,
2042 &mut attempt,
2043 dataset_record,
2044 &transformation,
2045 &inputs,
2046 DatasetMetadataRegistration {
2047 redcap_dictionary: None,
2048 variable_registrations: Vec::new(),
2049 loaded_column_names: Some(&loaded.column_names),
2050 force_metadata_updates: false,
2051 },
2052 )
2053 .await?;
2054 Ok::<_, AppError>((catalog, dataset, lineage, variables))
2055 }
2056 .await;
2057
2058 let (catalog, dataset, lineage, variables) = metadata_result?;
2059
2060 Ok(DatasetMaterialization {
2061 catalog,
2062 dataset,
2063 lineage,
2064 lake_relation: relation.qualified_name(),
2065 lake_schema: relation.schema_name,
2066 lake_table: relation.table_name,
2067 row_count: loaded.row_count,
2068 column_count: loaded.column_count,
2069 variables,
2070 metadata_warnings: Vec::new(),
2071 })
2072 }
2073 .await;
2074 drop(monitor);
2075 drop(metadata_deadline);
2076 attempt.finish(session, result)
2077 }
2078
2079 pub async fn export_dataset(
2080 &self,
2081 session: &mut DataStoreSession,
2082 request: ExportDatasetRequest,
2083 ) -> Result<ExportedDataset, AppError> {
2084 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Export
2085 {
2086 return Err(AppError::Validation(
2087 "dataset export requires an export transformation".to_string(),
2088 ));
2089 }
2090
2091 let dataset = self
2092 .assets
2093 .get_dataset(request.dataset_id)
2094 .await
2095 .map_err(|error| core_error("lookup dataset for export", error))?
2096 .ok_or_else(|| {
2097 AppError::Validation(format!(
2098 "dataset not found for export: {}",
2099 request.dataset_id.0
2100 ))
2101 })?;
2102 let catalog = self
2103 .catalog_for_version(request.dataset_id)
2104 .await
2105 .map_err(|error| core_error("lookup dataset asset version for export", error))?;
2106 if catalog.asset.asset_type != ahri_tre_types::AssetType::Dataset {
2107 return Err(AppError::Validation(format!(
2108 "version {} is not a dataset",
2109 request.dataset_id.0
2110 )));
2111 }
2112 let version = catalog
2113 .versions
2114 .iter()
2115 .find(|version| version.version_id == request.dataset_id)
2116 .ok_or_else(|| {
2117 AppError::Validation(format!(
2118 "asset version not found for dataset export: {}",
2119 request.dataset_id.0
2120 ))
2121 })?;
2122 let destination_path = export_destination_path(
2123 &request.target_dir,
2124 catalog.asset.name.as_str(),
2125 version,
2126 request.format.extension(),
2127 text_export_compressed(request.format, request.compress),
2128 )?;
2129 ensure_export_destination(&destination_path, request.overwrite)?;
2130 let arrow_metadata = match request.format {
2131 DatasetExportFormat::Parquet | DatasetExportFormat::Arrow => Some(
2132 self.dataset_arrow_metadata(&dataset, &catalog, version)
2133 .await?,
2134 ),
2135 DatasetExportFormat::Csv | DatasetExportFormat::Ndjson => None,
2136 };
2137
2138 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
2139 let exported = lake
2140 .export_dataset_table(
2141 session.lake_connection(),
2142 &ExportDatasetTableRequest {
2143 study_id: catalog.asset.study_id,
2144 dataset_name: catalog.asset.name.clone(),
2145 major: version.major,
2146 minor: version.minor,
2147 patch: version.patch,
2148 limit: request.limit,
2149 destination_path: destination_path.clone(),
2150 format: dataset_lake_export_format(request.format),
2151 compress: text_export_compressed(request.format, request.compress),
2152 arrow_metadata,
2153 },
2154 )
2155 .map_err(|error| infrastructure_error("export dataset from DuckLake", error))?;
2156 let lineage = self
2157 .record_export(RecordExportRequest {
2158 transformation: request.transformation,
2159 input_version_ids: vec![request.dataset_id],
2160 })
2161 .await?;
2162
2163 Ok(ExportedDataset {
2164 catalog,
2165 dataset,
2166 lineage,
2167 format: request.format,
2168 target_path: exported.destination_path,
2169 compressed: exported.compressed,
2170 row_count: exported.row_count,
2171 column_count: exported.column_count,
2172 exported_size_bytes: exported.exported_size_bytes,
2173 warnings: exported.warnings,
2174 })
2175 }
2176
2177 pub async fn export_dataset_to_path(
2178 &self,
2179 session: &mut DataStoreSession,
2180 request: ExportDatasetToPathRequest,
2181 ) -> Result<ExportedDataset, AppError> {
2182 if request.target_path.exists() && !request.overwrite {
2183 return Err(AppError::Validation(format!(
2184 "export destination already exists: {}",
2185 request.target_path.display()
2186 )));
2187 }
2188 let target_dir = request
2189 .target_path
2190 .parent()
2191 .map(Path::to_path_buf)
2192 .unwrap_or_else(|| PathBuf::from("."));
2193 let target_path = request.target_path.clone();
2194 let mut exported = self
2195 .export_dataset(
2196 session,
2197 ExportDatasetRequest {
2198 dataset_id: request.dataset_id,
2199 target_dir,
2200 format: request.format,
2201 limit: request.limit,
2202 compress: request.compress,
2203 overwrite: request.overwrite,
2204 transformation: request.transformation,
2205 },
2206 )
2207 .await?;
2208 if exported.target_path != target_path {
2209 if target_path.exists() && request.overwrite {
2210 fs::remove_file(&target_path).map_err(|error| {
2211 AppError::Infrastructure(format!(
2212 "remove existing dataset export target {}: {error}",
2213 target_path.display()
2214 ))
2215 })?;
2216 }
2217 fs::rename(&exported.target_path, &target_path).map_err(|error| {
2218 AppError::Infrastructure(format!(
2219 "move dataset export from {} to {}: {error}",
2220 exported.target_path.display(),
2221 target_path.display()
2222 ))
2223 })?;
2224 exported.target_path = target_path;
2225 }
2226 Ok(exported)
2227 }
2228
2229 pub async fn export_datafile(
2230 &self,
2231 session: &DataStoreSession,
2232 request: ExportDataFileRequest,
2233 ) -> Result<ExportedDataFile, AppError> {
2234 if request.transformation.transformation_type != ahri_tre_types::TransformationType::Export
2235 {
2236 return Err(AppError::Validation(
2237 "datafile export requires an export transformation".to_string(),
2238 ));
2239 }
2240
2241 let datafile = self
2242 .assets
2243 .get_datafile(request.datafile_id)
2244 .await
2245 .map_err(|error| core_error("lookup datafile for export", error))?
2246 .ok_or_else(|| {
2247 AppError::Validation(format!(
2248 "datafile not found for export: {}",
2249 request.datafile_id.0
2250 ))
2251 })?;
2252 let catalog = self
2253 .catalog_for_version(request.datafile_id)
2254 .await
2255 .map_err(|error| core_error("lookup datafile asset version for export", error))?;
2256 if catalog.asset.asset_type != ahri_tre_types::AssetType::File {
2257 return Err(AppError::Validation(format!(
2258 "version {} is not a datafile",
2259 request.datafile_id.0
2260 )));
2261 }
2262 let version = catalog
2263 .versions
2264 .iter()
2265 .find(|version| version.version_id == request.datafile_id)
2266 .ok_or_else(|| {
2267 AppError::Validation(format!(
2268 "asset version not found for datafile export: {}",
2269 request.datafile_id.0
2270 ))
2271 })?;
2272 let extension = datafile_logical_extension(&datafile);
2273 let destination_path = export_destination_path(
2274 &request.target_dir,
2275 catalog.asset.name.as_str(),
2276 version,
2277 &extension,
2278 request.compress,
2279 )?;
2280 ensure_export_destination(&destination_path, request.overwrite)?;
2281
2282 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
2283 let exported = lake
2284 .export_datafile(&ahri_tre_lake::ExportDataFileRequest {
2285 datafile: datafile.clone(),
2286 destination_path: destination_path.clone(),
2287 compress: request.compress,
2288 })
2289 .map_err(|error| infrastructure_error("export datafile from lake", error))?;
2290 let lineage = self
2291 .record_export(RecordExportRequest {
2292 transformation: request.transformation,
2293 input_version_ids: vec![request.datafile_id],
2294 })
2295 .await?;
2296
2297 Ok(ExportedDataFile {
2298 catalog,
2299 datafile,
2300 lineage,
2301 target_path: exported.destination_path,
2302 compressed: exported.compressed,
2303 stored_size_bytes: exported.stored_size_bytes,
2304 logical_size_bytes: exported.logical_size_bytes,
2305 exported_size_bytes: exported.exported_size_bytes,
2306 })
2307 }
2308
2309 pub async fn export_study_datafile(
2310 &self,
2311 session: &DataStoreSession,
2312 request: ExportStudyDataFileRequest,
2313 ) -> Result<ExportedDataFile, AppError> {
2314 let metadata = self
2315 .get_datafile_metadata(DataFileMetadataRequest {
2316 study_id: request.study_id,
2317 asset_name: request.asset_name,
2318 version: request.version,
2319 })
2320 .await?;
2321 self.export_datafile(
2322 session,
2323 ExportDataFileRequest {
2324 datafile_id: metadata.version.version_id,
2325 target_dir: request.target_dir,
2326 compress: request.compress,
2327 overwrite: request.overwrite,
2328 transformation: request.transformation,
2329 },
2330 )
2331 .await
2332 }
2333
2334 pub async fn register_domain_study(
2335 &self,
2336 request: RegisterDomainStudyRequest,
2337 ) -> Result<StudyRegistration, AppError> {
2338 let domain = self
2339 .register_domain(RegisterDomainRequest {
2340 domain: request.domain,
2341 })
2342 .await?;
2343
2344 self.register_study(RegisterStudyRequest {
2345 study: request.study,
2346 domain_ids: vec![domain.domain_id],
2347 })
2348 .await
2349 }
2350
2351 pub async fn register_dataset_variables(
2352 &self,
2353 request: RegisterDatasetVariablesRequest,
2354 ) -> Result<Vec<RegisteredDatasetVariable>, AppError> {
2355 self.assets
2356 .get_dataset(request.dataset_id)
2357 .await
2358 .map_err(|error| core_error("lookup dataset for variable registration", error))?
2359 .ok_or_else(|| {
2360 AppError::Validation(format!(
2361 "dataset not found for variable registration: {}",
2362 request.dataset_id.0
2363 ))
2364 })?;
2365
2366 self.register_dataset_variables_internal(
2367 request.dataset_id,
2368 Some(request.domain_id),
2369 request
2370 .variables
2371 .into_iter()
2372 .map(DatasetVariableRegistrationInput::from)
2373 .collect(),
2374 None,
2375 false,
2376 )
2377 .await
2378 }
2379
2380 pub async fn register_vocabulary_mapping(
2381 &self,
2382 request: RegisterVocabularyMappingRequest,
2383 ) -> Result<VocabularyMappingDetail, AppError> {
2384 let from_item = self
2385 .resolve_vocabulary_item_reference("from vocabulary item", &request.from)
2386 .await?;
2387 let to_item = self
2388 .resolve_vocabulary_item_reference("to vocabulary item", &request.to)
2389 .await?;
2390
2391 let mappings = self
2392 .vocabularies
2393 .list_vocabulary_mappings()
2394 .await
2395 .map_err(|error| core_error("list vocabulary mappings", error))?;
2396 let mapping = if let Some(existing) = mappings.iter().find(|mapping| {
2397 mapping.from_vocabulary_item == from_item.vocabulary_item_id
2398 && mapping.to_vocabulary_item == to_item.vocabulary_item_id
2399 }) {
2400 existing.clone()
2401 } else {
2402 self.vocabularies
2403 .create_vocabulary_mapping(&ahri_tre_types::NewVocabularyMappingRecord {
2404 from_vocabulary_item: from_item.vocabulary_item_id,
2405 to_vocabulary_item: to_item.vocabulary_item_id,
2406 })
2407 .await
2408 .map_err(|error| core_error("save vocabulary mapping", error))?
2409 };
2410 self.vocabulary_mapping_detail(mapping).await
2411 }
2412
2413 pub async fn list_vocabulary_mappings(
2414 &self,
2415 request: ListVocabularyMappingsRequest,
2416 ) -> Result<Vec<VocabularyMappingDetail>, AppError> {
2417 let mappings = self
2418 .vocabularies
2419 .list_vocabulary_mappings()
2420 .await
2421 .map_err(|error| core_error("list vocabulary mappings", error))?;
2422 let mut details = Vec::new();
2423 for mapping in mappings {
2424 let detail = self.vocabulary_mapping_detail(mapping).await?;
2425 if let Some(item_id) = request.item_id
2426 && detail.mapping.from_vocabulary_item != item_id
2427 && detail.mapping.to_vocabulary_item != item_id
2428 {
2429 continue;
2430 }
2431 if let Some(vocabulary_id) = request.from_vocabulary_id
2432 && detail.from_vocabulary.vocabulary_id != vocabulary_id
2433 {
2434 continue;
2435 }
2436 if let Some(vocabulary_id) = request.to_vocabulary_id
2437 && detail.to_vocabulary.vocabulary_id != vocabulary_id
2438 {
2439 continue;
2440 }
2441 if let Some(vocabulary_id) = request.involving_vocabulary_id
2442 && detail.from_vocabulary.vocabulary_id != vocabulary_id
2443 && detail.to_vocabulary.vocabulary_id != vocabulary_id
2444 {
2445 continue;
2446 }
2447 details.push(detail);
2448 }
2449 Ok(details)
2450 }
2451
2452 pub async fn vocabulary_mapping_summary(
2453 &self,
2454 request: VocabularyMappingSummaryRequest,
2455 ) -> Result<VocabularyMappingSummary, AppError> {
2456 let vocabulary = self
2457 .vocabularies
2458 .get_vocabulary(request.vocabulary_id)
2459 .await
2460 .map_err(|error| core_error("lookup vocabulary", error))?
2461 .ok_or_else(|| {
2462 AppError::Validation(format!("vocabulary {} not found", request.vocabulary_id.0))
2463 })?;
2464 let details = self
2465 .list_vocabulary_mappings(ListVocabularyMappingsRequest {
2466 from_vocabulary_id: None,
2467 to_vocabulary_id: None,
2468 involving_vocabulary_id: Some(request.vocabulary_id),
2469 item_id: None,
2470 })
2471 .await?;
2472 let mut peers = std::collections::BTreeMap::<i64, VocabularyMappingSummaryPeer>::new();
2473 let mut outgoing_count = 0;
2474 let mut incoming_count = 0;
2475 for detail in details {
2476 if detail.from_vocabulary.vocabulary_id == request.vocabulary_id {
2477 outgoing_count += 1;
2478 let peer = peers.entry(detail.to_vocabulary.vocabulary_id.0).or_insert(
2479 VocabularyMappingSummaryPeer {
2480 vocabulary: detail.to_vocabulary.clone(),
2481 outgoing_count: 0,
2482 incoming_count: 0,
2483 },
2484 );
2485 peer.outgoing_count += 1;
2486 }
2487 if detail.to_vocabulary.vocabulary_id == request.vocabulary_id {
2488 incoming_count += 1;
2489 let peer = peers
2490 .entry(detail.from_vocabulary.vocabulary_id.0)
2491 .or_insert(VocabularyMappingSummaryPeer {
2492 vocabulary: detail.from_vocabulary.clone(),
2493 outgoing_count: 0,
2494 incoming_count: 0,
2495 });
2496 peer.incoming_count += 1;
2497 }
2498 }
2499 Ok(VocabularyMappingSummary {
2500 vocabulary,
2501 has_mappings: outgoing_count > 0 || incoming_count > 0,
2502 outgoing_count,
2503 incoming_count,
2504 mapped_vocabularies: peers.into_values().collect(),
2505 })
2506 }
2507
2508 pub async fn register_variable(
2509 &self,
2510 request: RegisterVariableRequest,
2511 ) -> Result<RegisteredVariable, AppError> {
2512 self.domains
2513 .get_domain_by_id(request.domain_id)
2514 .await
2515 .map_err(|error| core_error("lookup variable domain", error))?
2516 .ok_or_else(|| {
2517 AppError::Validation(format!(
2518 "variable domain not found: {}",
2519 request.domain_id.0
2520 ))
2521 })?;
2522 validate_metadata_key_role("variable key_role", request.key_role)?;
2523 let value_type = self
2524 .resolve_value_type(request.value_type_id, request.value_type.as_deref())
2525 .await?;
2526 if let Some(vocabulary_id) = request.vocabulary_id {
2527 self.vocabularies
2528 .list_vocabularies(request.domain_id)
2529 .await
2530 .map_err(|error| core_error("list variable vocabularies", error))?
2531 .into_iter()
2532 .find(|vocabulary| vocabulary.vocabulary_id == vocabulary_id)
2533 .ok_or_else(|| {
2534 AppError::Validation(format!(
2535 "vocabulary {} not found in domain {}",
2536 vocabulary_id.0, request.domain_id.0
2537 ))
2538 })?;
2539 }
2540
2541 let extraction = ahri_tre_core::extract_inline_tags(request.description.as_deref());
2542 let variable = ahri_tre_types::NewVariableRecord {
2543 domain_id: request.domain_id,
2544 name: request.name,
2545 value_type_id: value_type.value_type_id,
2546 value_format: request.value_format,
2547 vocabulary_id: request.vocabulary_id,
2548 key_role: request.key_role,
2549 description: extraction.cleaned_text,
2550 note: request.note,
2551 ontology_namespace: request.ontology_namespace,
2552 ontology_class: request.ontology_class,
2553 };
2554 let variable = if let Some(variable_id) = request.variable_id {
2555 let variable = ahri_tre_types::VariableRecord {
2556 variable_id,
2557 domain_id: variable.domain_id,
2558 name: variable.name,
2559 value_type_id: variable.value_type_id,
2560 value_format: variable.value_format,
2561 vocabulary_id: variable.vocabulary_id,
2562 key_role: variable.key_role,
2563 description: variable.description,
2564 note: variable.note,
2565 ontology_namespace: variable.ontology_namespace,
2566 ontology_class: variable.ontology_class,
2567 };
2568 self.register_or_validate_variable(
2569 request.domain_id,
2570 &variable,
2571 request.force_metadata_updates,
2572 )
2573 .await?
2574 } else {
2575 self.register_or_validate_new_variable(
2576 request.domain_id,
2577 &variable,
2578 request.force_metadata_updates,
2579 )
2580 .await?
2581 };
2582 self.attach_tags(
2583 TagAttachmentTarget::Variable(variable.variable_id),
2584 extraction.tags,
2585 )
2586 .await?;
2587 self.registered_variable_metadata(variable).await
2588 }
2589
2590 pub async fn list_variables(
2591 &self,
2592 request: ListVariablesRequest,
2593 ) -> Result<Vec<RegisteredVariable>, AppError> {
2594 let domains = self
2595 .semantic_domains(request.domain, request.study, "variable scope")
2596 .await?;
2597
2598 let mut variables = Vec::new();
2599 for domain in domains {
2600 for variable in self
2601 .variables
2602 .list_domain_variables(domain.domain_id)
2603 .await
2604 .map_err(|error| core_error("list domain variables", error))?
2605 {
2606 variables.push(self.registered_variable_metadata(variable).await?);
2607 }
2608 }
2609 variables.sort_by(|left, right| {
2610 left.variable
2611 .domain_id
2612 .0
2613 .cmp(&right.variable.domain_id.0)
2614 .then_with(|| {
2615 left.variable
2616 .name
2617 .as_str()
2618 .cmp(right.variable.name.as_str())
2619 })
2620 });
2621 Ok(variables)
2622 }
2623
2624 pub async fn search_variables(
2625 &self,
2626 request: SearchVariablesRequest,
2627 ) -> Result<VariableSearchResult, AppError> {
2628 let query = app_variable_search_query(request)?;
2629 let page = self
2630 .variables
2631 .search_variables(&query)
2632 .await
2633 .map_err(|error| core_error("search variables", error))?;
2634 let mut variables = Vec::with_capacity(page.items.len());
2635 for entry in page.items {
2636 let metadata = self.registered_variable_metadata(entry.variable).await?;
2637 let mut tags = self
2638 .tags
2639 .list_tags_for_target(TagAttachmentTarget::Variable(metadata.variable.variable_id))
2640 .await
2641 .map_err(|error| core_error("list search variable tags", error))?;
2642 tags.truncate(MAX_SUMMARY_TAGS);
2643 variables.push(VariableSearchEntry {
2644 variable: metadata.variable,
2645 value_type: metadata.value_type,
2646 vocabulary: metadata.vocabulary,
2647 vocabulary_items: metadata.vocabulary_items,
2648 tags,
2649 });
2650 }
2651 Ok(VariableSearchResult {
2652 variables,
2653 next_cursor: page.next_cursor,
2654 })
2655 }
2656
2657 pub async fn get_variable(
2658 &self,
2659 request: GetVariableRequest,
2660 ) -> Result<Option<RegisteredVariable>, AppError> {
2661 match self
2662 .resolve_variable_selector_optional(request.variable)
2663 .await?
2664 {
2665 Some(variable) => Ok(Some(self.registered_variable_metadata(variable).await?)),
2666 None => Ok(None),
2667 }
2668 }
2669
2670 pub async fn add_variable(
2671 &self,
2672 request: AddVariableRequest,
2673 ) -> Result<RegisteredVariable, AppError> {
2674 self.add_or_update_variable(VariableCommandFields {
2675 domain: request.domain,
2676 name: request.name,
2677 value_type: Some(request.value_type),
2678 value_format: request.value_format,
2679 key_role: request.key_role,
2680 description: request.description,
2681 ontology_namespace: request.ontology_namespace,
2682 ontology_class: request.ontology_class,
2683 vocabulary: request.vocabulary,
2684 vocabulary_items: request.vocabulary_items,
2685 existing: None,
2686 force_metadata_updates: false,
2687 })
2688 .await
2689 }
2690
2691 pub async fn update_variable(
2692 &self,
2693 request: UpdateVariableRequest,
2694 ) -> Result<RegisteredVariable, AppError> {
2695 let label = variable_selector_label(&request.variable);
2696 let existing = self
2697 .resolve_variable_selector_optional(request.variable)
2698 .await?
2699 .ok_or_else(|| AppError::Validation(format!("variable not found: {label}")))?;
2700 self.add_or_update_variable(VariableCommandFields {
2701 domain: DomainSelector::Id {
2702 domain_id: existing.domain_id,
2703 },
2704 name: existing.name.as_str().to_string(),
2705 value_type: request.value_type,
2706 value_format: request.value_format,
2707 key_role: request.key_role,
2708 description: request.description,
2709 ontology_namespace: request.ontology_namespace,
2710 ontology_class: request.ontology_class,
2711 vocabulary: request.vocabulary,
2712 vocabulary_items: None,
2713 existing: Some(existing),
2714 force_metadata_updates: true,
2715 })
2716 .await
2717 }
2718
2719 pub async fn delete_variable(&self, request: GetVariableRequest) -> Result<(), AppError> {
2720 let _ = self
2721 .get_variable(request)
2722 .await?
2723 .ok_or_else(|| AppError::Validation("variable not found".to_string()))?;
2724 Err(AppError::NotImplemented(
2725 "variable delete is not supported by the semantic catalog repository",
2726 ))
2727 }
2728
2729 pub async fn list_vocabularies(
2730 &self,
2731 request: ListVocabulariesRequest,
2732 ) -> Result<Vec<RegisteredVocabulary>, AppError> {
2733 let domains = self
2734 .semantic_domains(request.domain, request.study, "vocabulary scope")
2735 .await?;
2736
2737 let mut registered = Vec::new();
2738 for domain in domains {
2739 let vocabularies = self
2740 .vocabularies
2741 .list_vocabularies(domain.domain_id)
2742 .await
2743 .map_err(|error| core_error("list vocabularies", error))?;
2744 for vocabulary in vocabularies {
2745 registered.push(self.registered_vocabulary_metadata(vocabulary).await?);
2746 }
2747 }
2748 registered.sort_by(|left, right| {
2749 left.vocabulary
2750 .domain_id
2751 .0
2752 .cmp(&right.vocabulary.domain_id.0)
2753 .then_with(|| {
2754 left.vocabulary
2755 .name
2756 .as_str()
2757 .cmp(right.vocabulary.name.as_str())
2758 })
2759 });
2760 Ok(registered)
2761 }
2762
2763 pub async fn get_vocabulary(
2764 &self,
2765 request: GetVocabularyRequest,
2766 ) -> Result<Option<RegisteredVocabulary>, AppError> {
2767 match self
2768 .resolve_vocabulary_selector_optional(request.vocabulary)
2769 .await?
2770 {
2771 Some(vocabulary) => Ok(Some(self.registered_vocabulary_metadata(vocabulary).await?)),
2772 None => Ok(None),
2773 }
2774 }
2775
2776 pub async fn add_vocabulary(
2777 &self,
2778 request: AddVocabularyRequest,
2779 ) -> Result<RegisteredVocabulary, AppError> {
2780 let domain = self
2781 .resolve_domain_selector(request.domain, "vocabulary domain")
2782 .await?;
2783 self.register_vocabulary(RegisterVocabularyRequest {
2784 domain_id: domain.domain_id,
2785 vocabulary_id: None,
2786 name: parse_vocabulary_name(request.name)?,
2787 description: request.description,
2788 items: parse_vocabulary_item_requests(&request.items)?,
2789 force_metadata_updates: false,
2790 })
2791 .await
2792 }
2793
2794 pub async fn delete_vocabulary(&self, request: GetVocabularyRequest) -> Result<(), AppError> {
2795 let _ = self
2796 .get_vocabulary(request)
2797 .await?
2798 .ok_or_else(|| AppError::Validation("vocabulary not found".to_string()))?;
2799 Err(AppError::NotImplemented(
2800 "vocabulary delete is not supported by the semantic catalog repository",
2801 ))
2802 }
2803
2804 pub async fn plan_semantic_delete(
2805 &self,
2806 request: PlanSemanticDeleteRequest,
2807 ) -> Result<SemanticDeletePlanningResult, AppError> {
2808 let reason = request.reason.trim().to_string();
2809 let actor = request
2810 .actor
2811 .as_deref()
2812 .map(str::trim)
2813 .filter(|actor| !actor.is_empty())
2814 .map(str::to_string);
2815 let mut validation_errors = Vec::new();
2816 if reason.is_empty() {
2817 validation_errors.push(semantic_delete_validation_error(
2818 "missing_reason",
2819 "semantic delete reason must not be empty",
2820 ));
2821 }
2822 if actor.is_none() {
2823 validation_errors.push(semantic_delete_validation_error(
2824 "missing_actor",
2825 "semantic delete planning requires an authenticated TRE user or explicit actor",
2826 ));
2827 }
2828 if !validation_errors.is_empty() {
2829 return Ok(SemanticDeletePlanningResult::rejected(validation_errors));
2830 }
2831
2832 let Some((identity, summary)) = self.resolve_semantic_delete_plan(&request.target).await?
2833 else {
2834 return Ok(SemanticDeletePlanningResult::rejected(vec![
2835 semantic_delete_validation_error(
2836 "target_not_found",
2837 "semantic delete target was not found",
2838 ),
2839 ]));
2840 };
2841 let disposition = if summary.has_blockers() {
2842 SemanticDeleteDisposition::Blocked
2843 } else {
2844 SemanticDeleteDisposition::PhysicalDeleteAllowed
2845 };
2846 let actor = actor.expect("actor checked");
2847 Ok(SemanticDeletePlanningResult::accepted(SemanticDeletePlan {
2848 target: request.target,
2849 target_identity: identity,
2850 authorization: SemanticDeleteAuthorization {
2851 status: SemanticDeleteAuthorizationStatus::ActorRecorded,
2852 principal: actor.clone(),
2853 requirement:
2854 "semantic-catalog physical deletion records the authenticated TRE user"
2855 .to_string(),
2856 },
2857 reason,
2858 actor,
2859 dependency_summary: summary,
2860 disposition,
2861 }))
2862 }
2863
2864 pub async fn execute_semantic_delete(
2865 &self,
2866 request: ExecuteSemanticDeleteRequest,
2867 ) -> Result<ExecuteSemanticDeleteResult, AppError> {
2868 if !request.confirmed {
2869 return Err(AppError::Validation(
2870 "semantic delete execution requires --yes; use --dry-run to inspect the plan"
2871 .to_string(),
2872 ));
2873 }
2874 if request.plan.disposition != SemanticDeleteDisposition::PhysicalDeleteAllowed {
2875 return Err(AppError::Validation(
2876 "semantic delete plan is dependency-blocked".to_string(),
2877 ));
2878 }
2879 let Some((_, fresh_summary)) = self
2880 .resolve_semantic_delete_plan(&request.plan.target)
2881 .await?
2882 else {
2883 return Ok(ExecuteSemanticDeleteResult {
2884 plan: request.plan,
2885 deleted: false,
2886 disposition: SemanticDeleteExecutionDisposition::AlreadyAbsent,
2887 });
2888 };
2889 if fresh_summary.has_blockers() {
2890 return Err(AppError::Validation(
2891 "semantic delete dependencies changed after planning; inspect a fresh dry-run"
2892 .to_string(),
2893 ));
2894 }
2895 let id = parse_semantic_i64_id(&request.plan.target_identity.id);
2896 let deleted = match request.plan.target_identity.kind {
2897 SemanticDeleteTargetKind::Domain => self
2898 .domains
2899 .delete_domain_record(ahri_tre_types::DomainId(id))
2900 .await
2901 .map_err(|error| core_error("delete domain record", error))?,
2902 SemanticDeleteTargetKind::Variable => self
2903 .variables
2904 .delete_variable_record(ahri_tre_types::VariableId(id))
2905 .await
2906 .map_err(|error| core_error("delete variable record", error))?,
2907 SemanticDeleteTargetKind::Vocabulary => self
2908 .vocabularies
2909 .delete_vocabulary_record(ahri_tre_types::VocabularyId(id))
2910 .await
2911 .map_err(|error| core_error("delete vocabulary record", error))?,
2912 SemanticDeleteTargetKind::Entity => self
2913 .entities
2914 .delete_entity_record(ahri_tre_types::EntityId(id))
2915 .await
2916 .map_err(|error| core_error("delete entity record", error))?,
2917 SemanticDeleteTargetKind::EntityRelation => self
2918 .entities
2919 .delete_entity_relation_record(ahri_tre_types::EntityRelationId(id))
2920 .await
2921 .map_err(|error| core_error("delete entity relation record", error))?,
2922 };
2923 Ok(ExecuteSemanticDeleteResult {
2924 plan: request.plan,
2925 deleted,
2926 disposition: if deleted {
2927 SemanticDeleteExecutionDisposition::PhysicallyDeleted
2928 } else {
2929 SemanticDeleteExecutionDisposition::AlreadyAbsent
2930 },
2931 })
2932 }
2933
2934 pub async fn get_dataset_variables(
2935 &self,
2936 request: GetDatasetVariablesRequest,
2937 ) -> Result<DatasetVariablesMetadata, AppError> {
2938 let dataset = self
2939 .assets
2940 .get_dataset(request.dataset_id)
2941 .await
2942 .map_err(|error| core_error("lookup dataset variables dataset", error))?
2943 .ok_or_else(|| {
2944 AppError::Validation(format!(
2945 "dataset not found for variable metadata: {}",
2946 request.dataset_id.0
2947 ))
2948 })?;
2949 let links = self
2950 .variables
2951 .list_dataset_variables(request.dataset_id)
2952 .await
2953 .map_err(|error| core_error("list dataset variable links", error))?;
2954 let mut variables = Vec::with_capacity(links.len());
2955 for link in links {
2956 variables.push(self.dataset_variable_metadata(link).await?);
2957 }
2958 Ok(DatasetVariablesMetadata { dataset, variables })
2959 }
2960
2961 pub async fn list_tags(&self) -> Result<Vec<ahri_tre_types::TagRecord>, AppError> {
2962 self.tags
2963 .list_tags()
2964 .await
2965 .map_err(|error| core_error("list tags", error))
2966 }
2967
2968 pub async fn get_tags(&self, request: GetTagsRequest) -> Result<TargetTags, AppError> {
2969 let tags = self
2970 .tags
2971 .list_tags_for_target(request.target.clone())
2972 .await
2973 .map_err(|error| core_error("list tags for target", error))?;
2974 Ok(TargetTags {
2975 target: request.target,
2976 tags,
2977 })
2978 }
2979
2980 pub async fn set_tags(&self, request: SetTagsRequest) -> Result<TargetTags, AppError> {
2981 let normalized = ahri_tre_core::normalize_tags(&request.tags);
2982 let tags = self
2983 .tags
2984 .set_tags_for_target(request.target.clone(), &normalized)
2985 .await
2986 .map_err(|error| core_error("set tags for target", error))?;
2987 Ok(TargetTags {
2988 target: request.target,
2989 tags,
2990 })
2991 }
2992
2993 pub async fn register_entity(
2994 &self,
2995 request: RegisterEntityRequest,
2996 ) -> Result<ahri_tre_types::EntityRecord, AppError> {
2997 self.domains
2998 .get_domain_by_id(request.entity.domain_id)
2999 .await
3000 .map_err(|error| core_error("lookup entity domain", error))?
3001 .ok_or_else(|| {
3002 AppError::Validation(format!(
3003 "entity domain not found: {}",
3004 request.entity.domain_id.0
3005 ))
3006 })?;
3007
3008 let extraction = ahri_tre_core::extract_inline_tags(request.entity.description.as_deref());
3009 let mut requested = request.entity;
3010 requested.description = extraction.cleaned_text;
3011
3012 let existing = self
3013 .entities
3014 .list_entities(requested.domain_id)
3015 .await
3016 .map_err(|error| core_error("list entities", error))?
3017 .into_iter()
3018 .find(|entity| entity.name == requested.name);
3019 let entity = match existing {
3020 Some(entity) => entity,
3021 None => self
3022 .entities
3023 .create_entity(&requested)
3024 .await
3025 .map_err(|error| core_error("create entity", error))?,
3026 };
3027 self.attach_tags(
3028 TagAttachmentTarget::Entity(entity.entity_id),
3029 extraction.tags,
3030 )
3031 .await?;
3032 Ok(entity)
3033 }
3034
3035 pub async fn register_entity_relation(
3036 &self,
3037 request: RegisterEntityRelationRequest,
3038 ) -> Result<ahri_tre_types::EntityRelationRecord, AppError> {
3039 self.domains
3040 .get_domain_by_id(request.relation.domain_id)
3041 .await
3042 .map_err(|error| core_error("lookup entity relation domain", error))?
3043 .ok_or_else(|| {
3044 AppError::Validation(format!(
3045 "entity relation domain not found: {}",
3046 request.relation.domain_id.0
3047 ))
3048 })?;
3049 self.require_entity_id(
3050 request.relation.domain_id,
3051 request.relation.subject_entity_id,
3052 )
3053 .await?;
3054 self.require_entity_id(
3055 request.relation.domain_id,
3056 request.relation.object_entity_id,
3057 )
3058 .await?;
3059
3060 let extraction =
3061 ahri_tre_core::extract_inline_tags(request.relation.description.as_deref());
3062 let mut requested = request.relation;
3063 requested.description = extraction.cleaned_text;
3064
3065 let existing = self
3066 .entities
3067 .list_entity_relations(requested.domain_id)
3068 .await
3069 .map_err(|error| core_error("list entity relations", error))?
3070 .into_iter()
3071 .find(|relation| relation.name == requested.name);
3072 let relation = match existing {
3073 Some(relation) => relation,
3074 None => self
3075 .entities
3076 .create_entity_relation(&requested)
3077 .await
3078 .map_err(|error| core_error("create entity relation", error))?,
3079 };
3080 self.attach_tags(
3081 TagAttachmentTarget::EntityRelation(relation.entity_relation_id),
3082 extraction.tags,
3083 )
3084 .await?;
3085 Ok(relation)
3086 }
3087
3088 pub async fn list_entities(
3089 &self,
3090 request: ListEntitiesRequest,
3091 ) -> Result<Vec<ahri_tre_types::EntityRecord>, AppError> {
3092 let domain = self
3093 .resolve_domain_selector(request.domain, "entity domain")
3094 .await?;
3095 self.entities
3096 .list_entities(domain.domain_id)
3097 .await
3098 .map_err(|error| core_error("list entities", error))
3099 }
3100
3101 pub async fn search_entities(
3102 &self,
3103 request: SearchEntitiesRequest,
3104 ) -> Result<EntitySearchResult, AppError> {
3105 let query = app_entity_search_query(request)?;
3106 let page = self
3107 .entities
3108 .search_entities(&query)
3109 .await
3110 .map_err(|error| core_error("search entities", error))?;
3111 let mut entities = Vec::with_capacity(page.items.len());
3112 for entry in page.items {
3113 let mut tags = self
3114 .tags
3115 .list_tags_for_target(TagAttachmentTarget::Entity(entry.entity.entity_id))
3116 .await
3117 .map_err(|error| core_error("list search entity tags", error))?;
3118 tags.truncate(MAX_SUMMARY_TAGS);
3119 entities.push(EntitySearchEntry {
3120 entity: entry.entity,
3121 tags,
3122 });
3123 }
3124 Ok(EntitySearchResult {
3125 entities,
3126 next_cursor: page.next_cursor,
3127 })
3128 }
3129
3130 pub async fn search_entity_relations(
3131 &self,
3132 request: SearchRelationsRequest,
3133 ) -> Result<RelationSearchResult, AppError> {
3134 let query = app_relation_search_query(request)?;
3135 let page = self
3136 .entities
3137 .search_entity_relations(&query)
3138 .await
3139 .map_err(|error| core_error("search entity relations", error))?;
3140 let mut relations = Vec::with_capacity(page.items.len());
3141 for entry in page.items {
3142 let mut tags = self
3143 .tags
3144 .list_tags_for_target(TagAttachmentTarget::EntityRelation(
3145 entry.relation.entity_relation_id,
3146 ))
3147 .await
3148 .map_err(|error| core_error("list search relation tags", error))?;
3149 tags.truncate(MAX_SUMMARY_TAGS);
3150 relations.push(RelationSearchEntry {
3151 relation: entry.relation,
3152 source_entity: entry.source_entity,
3153 target_entity: entry.target_entity,
3154 tags,
3155 });
3156 }
3157 Ok(RelationSearchResult {
3158 relations,
3159 next_cursor: page.next_cursor,
3160 })
3161 }
3162
3163 pub async fn get_entity(
3164 &self,
3165 request: GetEntityRequest,
3166 ) -> Result<Option<ahri_tre_types::EntityRecord>, AppError> {
3167 self.resolve_entity_selector_optional(request.entity).await
3168 }
3169
3170 pub async fn add_entity(
3171 &self,
3172 request: AddEntityRequest,
3173 ) -> Result<ahri_tre_types::EntityRecord, AppError> {
3174 let domain = self
3175 .resolve_domain_selector(request.domain, "entity domain")
3176 .await?;
3177 self.register_entity(RegisterEntityRequest {
3178 entity: ahri_tre_types::NewEntityRecord {
3179 domain_id: domain.domain_id,
3180 name: parse_entity_name(request.name)?,
3181 description: request.description,
3182 agent_instructions: request.agent_instructions,
3183 ontology_namespace: request.ontology_namespace,
3184 ontology_class: request.ontology_class,
3185 },
3186 })
3187 .await
3188 }
3189
3190 pub async fn list_entity_relations(
3191 &self,
3192 request: ListEntityRelationsRequest,
3193 ) -> Result<Vec<ahri_tre_types::EntityRelationRecord>, AppError> {
3194 let domain = self
3195 .resolve_domain_selector(request.domain, "entity relation domain")
3196 .await?;
3197 self.entities
3198 .list_entity_relations(domain.domain_id)
3199 .await
3200 .map_err(|error| core_error("list entity relations", error))
3201 }
3202
3203 pub async fn get_entity_relation(
3204 &self,
3205 request: GetEntityRelationRequest,
3206 ) -> Result<Option<ahri_tre_types::EntityRelationRecord>, AppError> {
3207 self.resolve_entity_relation_selector_optional(request.relation)
3208 .await
3209 }
3210
3211 pub async fn add_entity_relation(
3212 &self,
3213 request: AddEntityRelationRequest,
3214 ) -> Result<ahri_tre_types::EntityRelationRecord, AppError> {
3215 let domain = self
3216 .resolve_domain_selector(request.domain, "entity relation domain")
3217 .await?;
3218 let subject = self
3219 .require_entity_by_name(domain.domain_id, request.subject)
3220 .await?;
3221 let object = self
3222 .require_entity_by_name(domain.domain_id, request.object)
3223 .await?;
3224 self.register_entity_relation(RegisterEntityRelationRequest {
3225 relation: ahri_tre_types::NewEntityRelationRecord {
3226 subject_entity_id: subject.entity_id,
3227 object_entity_id: object.entity_id,
3228 domain_id: domain.domain_id,
3229 name: parse_entity_relation_name(request.name)?,
3230 description: request.description,
3231 agent_instructions: request.agent_instructions,
3232 ontology_namespace: request.ontology_namespace,
3233 ontology_class: request.ontology_class,
3234 },
3235 })
3236 .await
3237 }
3238
3239 pub async fn get_entity_instance(
3240 &self,
3241 request: GetEntityInstanceRequest,
3242 ) -> Result<Option<ahri_tre_types::EntityInstanceRecord>, AppError> {
3243 self.entities
3244 .get_entity_instance(request.instance_id)
3245 .await
3246 .map_err(|error| core_error("get entity instance", error))
3247 }
3248
3249 pub async fn list_entity_instances(
3250 &self,
3251 request: ListEntityInstancesRequest,
3252 ) -> Result<Vec<ahri_tre_types::EntityInstanceRecord>, AppError> {
3253 let domain = self.require_domain_by_name(request.domain).await?;
3254 let entity = self
3255 .require_entity_by_name(domain.domain_id, request.entity)
3256 .await?;
3257 self.entities
3258 .list_entity_instances(entity.entity_id)
3259 .await
3260 .map_err(|error| core_error("list entity instances", error))
3261 }
3262
3263 pub async fn add_entity_instance(
3264 &self,
3265 request: AddEntityInstanceRequest,
3266 ) -> Result<ahri_tre_types::EntityInstanceRecord, AppError> {
3267 require_positive_transformation_id(request.transformation_id)?;
3268 let domain = self.require_domain_by_name(request.domain).await?;
3269 let entity = self
3270 .require_entity_by_name(domain.domain_id, request.entity)
3271 .await?;
3272 self.entities
3273 .create_entity_instance(&ahri_tre_types::NewEntityInstanceRecord {
3274 entity_id: entity.entity_id,
3275 label: request.label,
3276 note: request.note,
3277 transformation_id: request.transformation_id,
3278 })
3279 .await
3280 .map_err(|error| core_error("create entity instance", error))
3281 }
3282
3283 pub async fn add_study_entity_instance_mapping(
3284 &self,
3285 request: AddStudyEntityInstanceMappingRequest,
3286 ) -> Result<EntityInstanceMapping, AppError> {
3287 require_positive_transformation_id(request.transformation_id)?;
3288 let study = self.require_study_by_name(request.study).await?;
3289 let instance = self
3290 .entities
3291 .get_entity_instance(request.instance_id)
3292 .await
3293 .map_err(|error| core_error("lookup entity instance", error))?
3294 .ok_or_else(|| {
3295 AppError::Validation(format!(
3296 "entity instance not found: {}",
3297 request.instance_id
3298 ))
3299 })?;
3300 let mapping = self
3301 .entities
3302 .save_study_entity_instance(&ahri_tre_types::StudyEntityInstanceWriteRecord {
3303 study_id: study.study_id,
3304 entity_instance_id: instance.instance_id,
3305 entity_id: instance.entity_id,
3306 external_id: request.external_id,
3307 transformation_id: request.transformation_id,
3308 })
3309 .await
3310 .map_err(|error| core_error("save study entity instance", error))?;
3311 Ok(EntityInstanceMapping { instance, mapping })
3312 }
3313
3314 pub async fn get_study_entity_instance_mapping(
3315 &self,
3316 request: GetStudyEntityInstanceMappingRequest,
3317 ) -> Result<Option<EntityInstanceMapping>, AppError> {
3318 let study = self.require_study_by_name(request.study).await?;
3319 let domain = self.require_domain_by_name(request.domain).await?;
3320 let entity = self
3321 .require_entity_by_name(domain.domain_id, request.entity)
3322 .await?;
3323 let mapping = self
3324 .entities
3325 .get_study_entity_instance(study.study_id, entity.entity_id, &request.external_id)
3326 .await
3327 .map_err(|error| core_error("lookup study entity instance", error))?;
3328 match mapping {
3329 Some(mapping) => {
3330 let instance = self
3331 .entities
3332 .get_entity_instance(mapping.entity_instance_id)
3333 .await
3334 .map_err(|error| core_error("lookup mapped entity instance", error))?
3335 .ok_or_else(|| {
3336 AppError::Validation(format!(
3337 "study entity mapping references missing entity instance: {}",
3338 mapping.entity_instance_id
3339 ))
3340 })?;
3341 Ok(Some(EntityInstanceMapping { instance, mapping }))
3342 }
3343 None => Ok(None),
3344 }
3345 }
3346
3347 pub async fn list_study_entity_instance_mappings(
3348 &self,
3349 request: ListStudyEntityInstanceMappingsRequest,
3350 ) -> Result<EntityInstanceMappingList, AppError> {
3351 if request.domain.is_some() != request.entity.is_some() {
3352 return Err(AppError::Validation(
3353 "domain and entity filters must be supplied together".to_string(),
3354 ));
3355 }
3356 let study = self.require_study_by_name(request.study).await?;
3357 let entity_filter = match (request.domain, request.entity) {
3358 (Some(domain), Some(entity)) => {
3359 let domain = self.require_domain_by_name(domain).await?;
3360 let entity = self
3361 .require_entity_by_name(domain.domain_id, entity)
3362 .await?;
3363 Some((domain, entity))
3364 }
3365 _ => None,
3366 };
3367 let mappings = self
3368 .entities
3369 .list_study_entity_instances(study.study_id)
3370 .await
3371 .map_err(|error| core_error("list study entity instances", error))?;
3372 let mut readbacks = Vec::new();
3373 for mapping in mappings {
3374 if let Some((_, entity)) = entity_filter.as_ref()
3375 && mapping.entity_id != entity.entity_id
3376 {
3377 continue;
3378 }
3379 let instance = self
3380 .require_entity_instance_id(mapping.entity_instance_id)
3381 .await?;
3382 let entity = match entity_filter.as_ref() {
3383 Some((_, entity)) => entity.clone(),
3384 None => self.require_entity_record(instance.entity_id).await?,
3385 };
3386 let domain = match entity_filter.as_ref() {
3387 Some((domain, _)) => domain.clone(),
3388 None => self.require_domain_id(entity.domain_id).await?,
3389 };
3390 readbacks.push(EntityInstanceMappingReadback {
3391 study: study.clone(),
3392 domain,
3393 entity,
3394 instance,
3395 mapping,
3396 });
3397 }
3398 Ok(EntityInstanceMappingList {
3399 study,
3400 mappings: readbacks,
3401 })
3402 }
3403
3404 pub async fn add_entity_asset_link(
3405 &self,
3406 request: AddEntityAssetLinkRequest,
3407 ) -> Result<ahri_tre_types::AssetVersionEntityRecord, AppError> {
3408 require_positive_transformation_id(request.transformation_id)?;
3409 self.entities
3410 .get_entity_instance(request.instance_id)
3411 .await
3412 .map_err(|error| core_error("lookup entity instance", error))?
3413 .ok_or_else(|| {
3414 AppError::Validation(format!(
3415 "entity instance not found: {}",
3416 request.instance_id
3417 ))
3418 })?;
3419 let version = self
3420 .resolve_study_asset_version(request.study, request.asset, request.version)
3421 .await?;
3422 let link = ahri_tre_types::AssetVersionEntityRecord {
3423 version_id: version.version_id,
3424 entity_instance_id: request.instance_id,
3425 transformation_id: request.transformation_id,
3426 };
3427 self.entities
3428 .link_asset_version_entity(&link)
3429 .await
3430 .map_err(|error| core_error("link asset version entity", error))
3431 }
3432
3433 pub async fn list_entity_asset_links(
3434 &self,
3435 request: ListEntityAssetLinksRequest,
3436 ) -> Result<EntityAssetLinkList, AppError> {
3437 match (request.instance_id, request.asset) {
3438 (Some(instance_id), None) => {
3439 let instance = self.require_entity_instance_id(instance_id).await?;
3440 let links = self
3441 .entities
3442 .list_asset_version_entities_for_instance(instance.instance_id)
3443 .await
3444 .map_err(|error| {
3445 core_error("list asset version entity links for instance", error)
3446 })?;
3447 let mut readbacks = Vec::new();
3448 for link in links {
3449 readbacks.push(self.entity_asset_link_readback(link).await?);
3450 }
3451 Ok(EntityAssetLinkList {
3452 resolved_asset: None,
3453 resolved_version: None,
3454 links: readbacks,
3455 })
3456 }
3457 (None, Some(asset)) => {
3458 let (_, catalog, version) = self
3459 .resolve_asset_version_by_optional_study(
3460 request.study,
3461 asset,
3462 request.version,
3463 None,
3464 )
3465 .await?;
3466 let links = self
3467 .entities
3468 .list_asset_version_entities(version.version_id)
3469 .await
3470 .map_err(|error| core_error("list asset version entity links", error))?;
3471 let mut readbacks = Vec::new();
3472 for link in links {
3473 readbacks.push(self.entity_asset_link_readback(link).await?);
3474 }
3475 Ok(EntityAssetLinkList {
3476 resolved_asset: Some(catalog.asset),
3477 resolved_version: Some(version),
3478 links: readbacks,
3479 })
3480 }
3481 (Some(_), Some(_)) => Err(AppError::Validation(
3482 "list entity asset links by either instance_id or asset, not both".to_string(),
3483 )),
3484 (None, None) => Err(AppError::Validation(
3485 "list entity asset links requires instance_id or asset".to_string(),
3486 )),
3487 }
3488 }
3489
3490 pub async fn add_entity_dataset_link(
3491 &self,
3492 request: AddEntityDatasetLinkRequest,
3493 ) -> Result<ahri_tre_types::DatasetVersionEntityRecord, AppError> {
3494 require_positive_transformation_id(request.transformation_id)?;
3495 self.entities
3496 .get_entity_instance(request.instance_id)
3497 .await
3498 .map_err(|error| core_error("lookup entity instance", error))?
3499 .ok_or_else(|| {
3500 AppError::Validation(format!(
3501 "entity instance not found: {}",
3502 request.instance_id
3503 ))
3504 })?;
3505 let (version, variables) = self
3506 .resolve_study_dataset_version_and_variables(
3507 request.study,
3508 request.dataset,
3509 request.version,
3510 )
3511 .await?;
3512 let variable_id = require_dataset_variable_id(&variables, &request.variable)?;
3513 let link = ahri_tre_types::DatasetVersionEntityRecord {
3514 version_id: version.version_id,
3515 entity_instance_id: request.instance_id,
3516 entity_variable_id: variable_id,
3517 transformation_id: request.transformation_id,
3518 };
3519 self.entities
3520 .link_dataset_version_entity(&link)
3521 .await
3522 .map_err(|error| core_error("link dataset version entity", error))
3523 }
3524
3525 pub async fn list_entity_instance_datasets(
3526 &self,
3527 request: ListEntityInstanceDatasetsRequest,
3528 ) -> Result<EntityDatasetLinkList, AppError> {
3529 let instance = self.require_entity_instance_id(request.instance_id).await?;
3530 let links = self
3531 .entities
3532 .list_dataset_version_entities_for_instance(instance.instance_id)
3533 .await
3534 .map_err(|error| core_error("list dataset version entity links for instance", error))?;
3535 let mut readbacks = Vec::new();
3536 for link in links {
3537 readbacks.push(self.entity_dataset_link_readback(link).await?);
3538 }
3539 Ok(EntityDatasetLinkList {
3540 resolved_dataset: None,
3541 resolved_version: None,
3542 links: readbacks,
3543 })
3544 }
3545
3546 pub async fn get_entity_dataset_link(
3547 &self,
3548 request: GetEntityDatasetLinkRequest,
3549 ) -> Result<Option<EntityDatasetLinkReadback>, AppError> {
3550 let instance = self.require_entity_instance_id(request.instance_id).await?;
3551 let (_, _, version) = self
3552 .resolve_asset_version_by_optional_study(
3553 request.study,
3554 request.dataset,
3555 request.version,
3556 Some(ahri_tre_types::AssetType::Dataset),
3557 )
3558 .await?;
3559 let links = self
3560 .entities
3561 .list_dataset_version_entities(version.version_id)
3562 .await
3563 .map_err(|error| core_error("list dataset version entity links", error))?;
3564 if let Some(link) = links
3565 .into_iter()
3566 .find(|link| link.entity_instance_id == instance.instance_id)
3567 {
3568 Ok(Some(self.entity_dataset_link_readback(link).await?))
3569 } else {
3570 Ok(None)
3571 }
3572 }
3573
3574 pub async fn list_entity_dataset_links(
3575 &self,
3576 request: ListEntityDatasetLinksRequest,
3577 ) -> Result<EntityDatasetLinkList, AppError> {
3578 let (_, catalog, version) = self
3579 .resolve_asset_version_by_optional_study(
3580 request.study,
3581 request.dataset,
3582 request.version,
3583 Some(ahri_tre_types::AssetType::Dataset),
3584 )
3585 .await?;
3586 let links = self
3587 .entities
3588 .list_dataset_version_entities(version.version_id)
3589 .await
3590 .map_err(|error| core_error("list dataset version entity links", error))?;
3591 let mut readbacks = Vec::new();
3592 for link in links {
3593 readbacks.push(self.entity_dataset_link_readback(link).await?);
3594 }
3595 Ok(EntityDatasetLinkList {
3596 resolved_dataset: Some(catalog.asset),
3597 resolved_version: Some(version),
3598 links: readbacks,
3599 })
3600 }
3601
3602 pub async fn ensure_entity_instances_from_dataset(
3603 &self,
3604 session: &DataStoreSession,
3605 request: EnsureEntityInstancesFromDatasetRequest,
3606 ) -> Result<EntityInstanceDatasetBatchResult, AppError> {
3607 let study = self.require_study_by_name(request.study.clone()).await?;
3608 let domain = self.require_domain_by_name(request.domain.clone()).await?;
3609 let entity = self
3610 .require_entity_by_name(domain.domain_id, request.entity.clone())
3611 .await?;
3612 let (version, variables) = self
3613 .resolve_study_dataset_version_and_variables(
3614 request.study.clone(),
3615 request.dataset.clone(),
3616 request.version.clone(),
3617 )
3618 .await?;
3619 let external_variable_id =
3620 require_dataset_variable_id(&variables, &request.external_id_variable)?;
3621 let template_columns = label_template_columns(request.label_template.as_deref())?;
3622 for column in &template_columns {
3623 require_dataset_variable_id(&variables, column)?;
3624 }
3625
3626 let dataset_name =
3627 ahri_tre_types::NcName::parse(request.dataset.clone()).map_err(|error| {
3628 AppError::Validation(format!("invalid dataset name {}: {error}", request.dataset))
3629 })?;
3630 let rows = read_entity_batch_source_rows(
3631 session,
3632 study.study_id,
3633 dataset_name.as_str(),
3634 &version,
3635 &request.external_id_variable,
3636 &template_columns,
3637 )?;
3638 self.apply_entity_batch(ResolvedEntityBatch {
3639 inputs: vec![version.version_id],
3640 request,
3641 study,
3642 entity,
3643 version,
3644 external_variable_id,
3645 rows,
3646 })
3647 .await
3648 }
3649
3650 pub async fn ensure_relation_instances_from_dataset(
3651 &self,
3652 session: &DataStoreSession,
3653 request: EnsureRelationInstancesFromDatasetRequest,
3654 ) -> Result<RelationInstanceDatasetBatchResult, AppError> {
3655 let study = self.require_study_by_name(request.study.clone()).await?;
3656 let domain = self.require_domain_by_name(request.domain.clone()).await?;
3657 let relation = self
3658 .require_entity_relation_by_name(domain.domain_id, request.relation.clone())
3659 .await?;
3660 let (version, variables) = self
3661 .resolve_study_dataset_version_and_variables(
3662 request.study.clone(),
3663 request.dataset.clone(),
3664 request.version.clone(),
3665 )
3666 .await?;
3667 let relation_variable_id =
3668 require_dataset_variable_id(&variables, &request.relation_external_id_variable)?;
3669 let subject_variable_id =
3670 require_dataset_variable_id(&variables, &request.subject_external_id_variable)?;
3671 let object_variable_id =
3672 require_dataset_variable_id(&variables, &request.object_external_id_variable)?;
3673 if let Some(variable) = request.valid_from_variable.as_ref() {
3674 require_dataset_variable_id(&variables, variable)?;
3675 }
3676 if let Some(variable) = request.valid_to_variable.as_ref() {
3677 require_dataset_variable_id(&variables, variable)?;
3678 }
3679
3680 let dataset_name =
3681 ahri_tre_types::NcName::parse(request.dataset.clone()).map_err(|error| {
3682 AppError::Validation(format!("invalid dataset name {}: {error}", request.dataset))
3683 })?;
3684 let rows = read_relation_batch_source_rows(
3685 session,
3686 study.study_id,
3687 dataset_name.as_str(),
3688 &version,
3689 &request,
3690 )?;
3691 self.apply_relation_batch(ResolvedRelationBatch {
3692 inputs: vec![version.version_id],
3693 request,
3694 study,
3695 relation,
3696 version,
3697 relation_variable_id,
3698 subject_variable_id,
3699 object_variable_id,
3700 rows,
3701 })
3702 .await
3703 }
3704
3705 pub async fn get_relation_instance(
3706 &self,
3707 request: GetRelationInstanceRequest,
3708 ) -> Result<Option<ahri_tre_types::RelationInstanceRecord>, AppError> {
3709 self.entities
3710 .get_relation_instance(request.relation_instance_id)
3711 .await
3712 .map_err(|error| core_error("get relation instance", error))
3713 }
3714
3715 pub async fn list_relation_instances(
3716 &self,
3717 request: ListRelationInstancesRequest,
3718 ) -> Result<Vec<ahri_tre_types::RelationInstanceRecord>, AppError> {
3719 let domain = self.require_domain_by_name(request.domain).await?;
3720 let relation = self
3721 .require_entity_relation_by_name(domain.domain_id, request.name)
3722 .await?;
3723 self.entities
3724 .list_relation_instances(relation.entity_relation_id)
3725 .await
3726 .map_err(|error| core_error("list relation instances", error))
3727 }
3728
3729 pub async fn add_relation_instance(
3730 &self,
3731 request: AddRelationInstanceRequest,
3732 ) -> Result<ahri_tre_types::RelationInstanceRecord, AppError> {
3733 require_positive_transformation_id(request.transformation_id)?;
3734 let domain = self.require_domain_by_name(request.domain).await?;
3735 let relation = self
3736 .require_entity_relation_by_name(domain.domain_id, request.name)
3737 .await?;
3738 self.entities
3739 .get_entity_instance(request.subject_instance_id)
3740 .await
3741 .map_err(|error| core_error("lookup relation subject instance", error))?
3742 .ok_or_else(|| {
3743 AppError::Validation(format!(
3744 "relation subject entity instance not found: {}",
3745 request.subject_instance_id
3746 ))
3747 })?;
3748 self.entities
3749 .get_entity_instance(request.object_instance_id)
3750 .await
3751 .map_err(|error| core_error("lookup relation object instance", error))?
3752 .ok_or_else(|| {
3753 AppError::Validation(format!(
3754 "relation object entity instance not found: {}",
3755 request.object_instance_id
3756 ))
3757 })?;
3758 self.entities
3759 .create_relation_instance(&ahri_tre_types::NewRelationInstanceRecord {
3760 entity_relation_id: relation.entity_relation_id,
3761 entity_instance_id_1: request.subject_instance_id,
3762 entity_instance_id_2: request.object_instance_id,
3763 valid_from: request.valid_from,
3764 valid_to: request.valid_to,
3765 note: request.note,
3766 transformation_id: request.transformation_id,
3767 })
3768 .await
3769 .map_err(|error| core_error("create relation instance", error))
3770 }
3771
3772 pub async fn add_study_relation_instance_mapping(
3773 &self,
3774 request: AddStudyRelationInstanceMappingRequest,
3775 ) -> Result<RelationInstanceMapping, AppError> {
3776 require_positive_transformation_id(request.transformation_id)?;
3777 let study = self.require_study_by_name(request.study).await?;
3778 let instance = self
3779 .entities
3780 .get_relation_instance(request.relation_instance_id)
3781 .await
3782 .map_err(|error| core_error("lookup relation instance", error))?
3783 .ok_or_else(|| {
3784 AppError::Validation(format!(
3785 "relation instance not found: {}",
3786 request.relation_instance_id
3787 ))
3788 })?;
3789 let mapping = self
3790 .entities
3791 .save_study_relation_instance(&ahri_tre_types::StudyRelationInstanceWriteRecord {
3792 study_id: study.study_id,
3793 relation_instance_id: instance.relation_instance_id,
3794 entity_relation_id: instance.entity_relation_id,
3795 external_id: request.external_id,
3796 transformation_id: request.transformation_id,
3797 })
3798 .await
3799 .map_err(|error| core_error("save study relation instance", error))?;
3800 Ok(RelationInstanceMapping { instance, mapping })
3801 }
3802
3803 pub async fn get_study_relation_instance_mapping(
3804 &self,
3805 request: GetStudyRelationInstanceMappingRequest,
3806 ) -> Result<Option<RelationInstanceMapping>, AppError> {
3807 let study = self.require_study_by_name(request.study).await?;
3808 let domain = self.require_domain_by_name(request.domain).await?;
3809 let relation = self
3810 .require_entity_relation_by_name(domain.domain_id, request.name)
3811 .await?;
3812 let mapping = self
3813 .entities
3814 .get_study_relation_instance(
3815 study.study_id,
3816 relation.entity_relation_id,
3817 &request.external_id,
3818 )
3819 .await
3820 .map_err(|error| core_error("lookup study relation instance", error))?;
3821 match mapping {
3822 Some(mapping) => {
3823 let instance = self
3824 .entities
3825 .get_relation_instance(mapping.relation_instance_id)
3826 .await
3827 .map_err(|error| core_error("lookup mapped relation instance", error))?
3828 .ok_or_else(|| {
3829 AppError::Validation(format!(
3830 "study relation mapping references missing relation instance: {}",
3831 mapping.relation_instance_id
3832 ))
3833 })?;
3834 Ok(Some(RelationInstanceMapping { instance, mapping }))
3835 }
3836 None => Ok(None),
3837 }
3838 }
3839
3840 pub async fn list_study_relation_instance_mappings(
3841 &self,
3842 request: ListStudyRelationInstanceMappingsRequest,
3843 ) -> Result<RelationInstanceMappingList, AppError> {
3844 if request.domain.is_some() != request.relation.is_some() {
3845 return Err(AppError::Validation(
3846 "domain and relation filters must be supplied together".to_string(),
3847 ));
3848 }
3849 let study = self.require_study_by_name(request.study).await?;
3850 let relation_filter = match (request.domain, request.relation) {
3851 (Some(domain), Some(relation)) => {
3852 let domain = self.require_domain_by_name(domain).await?;
3853 let relation = self
3854 .require_entity_relation_by_name(domain.domain_id, relation)
3855 .await?;
3856 Some((domain, relation))
3857 }
3858 _ => None,
3859 };
3860 let mappings = self
3861 .entities
3862 .list_study_relation_instances(study.study_id)
3863 .await
3864 .map_err(|error| core_error("list study relation instances", error))?;
3865 let mut readbacks = Vec::new();
3866 for mapping in mappings {
3867 if let Some((_, relation)) = relation_filter.as_ref()
3868 && mapping.entity_relation_id != relation.entity_relation_id
3869 {
3870 continue;
3871 }
3872 let instance = self
3873 .require_relation_instance_id(mapping.relation_instance_id)
3874 .await?;
3875 let relation = match relation_filter.as_ref() {
3876 Some((_, relation)) => relation.clone(),
3877 None => {
3878 self.require_entity_relation_record(instance.entity_relation_id)
3879 .await?
3880 }
3881 };
3882 let domain = match relation_filter.as_ref() {
3883 Some((domain, _)) => domain.clone(),
3884 None => self.require_domain_id(relation.domain_id).await?,
3885 };
3886 let subject_entity = self
3887 .require_entity_record(relation.subject_entity_id)
3888 .await?;
3889 let object_entity = self
3890 .require_entity_record(relation.object_entity_id)
3891 .await?;
3892 let subject_instance = self
3893 .require_entity_instance_id(instance.entity_instance_id_1)
3894 .await?;
3895 let object_instance = self
3896 .require_entity_instance_id(instance.entity_instance_id_2)
3897 .await?;
3898 readbacks.push(RelationInstanceMappingReadback {
3899 study: study.clone(),
3900 domain,
3901 relation,
3902 subject_entity,
3903 object_entity,
3904 subject_instance,
3905 object_instance,
3906 instance,
3907 mapping,
3908 });
3909 }
3910 Ok(RelationInstanceMappingList {
3911 study,
3912 mappings: readbacks,
3913 })
3914 }
3915
3916 pub async fn add_relation_asset_link(
3917 &self,
3918 request: AddRelationAssetLinkRequest,
3919 ) -> Result<ahri_tre_types::AssetVersionRelationInstanceRecord, AppError> {
3920 require_positive_transformation_id(request.transformation_id)?;
3921 self.entities
3922 .get_relation_instance(request.relation_instance_id)
3923 .await
3924 .map_err(|error| core_error("lookup relation instance", error))?
3925 .ok_or_else(|| {
3926 AppError::Validation(format!(
3927 "relation instance not found: {}",
3928 request.relation_instance_id
3929 ))
3930 })?;
3931 let version = self
3932 .resolve_study_asset_version(request.study, request.asset, request.version)
3933 .await?;
3934 let link = ahri_tre_types::AssetVersionRelationInstanceRecord {
3935 version_id: version.version_id,
3936 relation_instance_id: request.relation_instance_id,
3937 transformation_id: request.transformation_id,
3938 };
3939 self.entities
3940 .link_asset_version_relation_instance(&link)
3941 .await
3942 .map_err(|error| core_error("link asset version relation instance", error))
3943 }
3944
3945 pub async fn list_relation_asset_links(
3946 &self,
3947 request: ListRelationAssetLinksRequest,
3948 ) -> Result<RelationAssetLinkList, AppError> {
3949 match (request.relation_instance_id, request.asset) {
3950 (Some(relation_instance_id), None) => {
3951 let instance = self
3952 .require_relation_instance_id(relation_instance_id)
3953 .await?;
3954 let links = self
3955 .entities
3956 .list_asset_version_relation_instances_for_instance(
3957 instance.relation_instance_id,
3958 )
3959 .await
3960 .map_err(|error| {
3961 core_error("list asset version relation links for instance", error)
3962 })?;
3963 let mut readbacks = Vec::new();
3964 for link in links {
3965 readbacks.push(self.relation_asset_link_readback(link).await?);
3966 }
3967 Ok(RelationAssetLinkList {
3968 resolved_asset: None,
3969 resolved_version: None,
3970 links: readbacks,
3971 })
3972 }
3973 (None, Some(asset)) => {
3974 let (_, catalog, version) = self
3975 .resolve_asset_version_by_optional_study(
3976 request.study,
3977 asset,
3978 request.version,
3979 None,
3980 )
3981 .await?;
3982 let links = self
3983 .entities
3984 .list_asset_version_relation_instances(version.version_id)
3985 .await
3986 .map_err(|error| core_error("list asset version relation links", error))?;
3987 let mut readbacks = Vec::new();
3988 for link in links {
3989 readbacks.push(self.relation_asset_link_readback(link).await?);
3990 }
3991 Ok(RelationAssetLinkList {
3992 resolved_asset: Some(catalog.asset),
3993 resolved_version: Some(version),
3994 links: readbacks,
3995 })
3996 }
3997 (Some(_), Some(_)) => Err(AppError::Validation(
3998 "list relation asset links by either relation_instance_id or asset, not both"
3999 .to_string(),
4000 )),
4001 (None, None) => Err(AppError::Validation(
4002 "list relation asset links requires relation_instance_id or asset".to_string(),
4003 )),
4004 }
4005 }
4006
4007 pub async fn add_relation_dataset_link(
4008 &self,
4009 request: AddRelationDatasetLinkRequest,
4010 ) -> Result<ahri_tre_types::DatasetVersionRelationInstanceRecord, AppError> {
4011 require_positive_transformation_id(request.transformation_id)?;
4012 self.entities
4013 .get_relation_instance(request.relation_instance_id)
4014 .await
4015 .map_err(|error| core_error("lookup relation instance", error))?
4016 .ok_or_else(|| {
4017 AppError::Validation(format!(
4018 "relation instance not found: {}",
4019 request.relation_instance_id
4020 ))
4021 })?;
4022 let (version, variables) = self
4023 .resolve_study_dataset_version_and_variables(
4024 request.study,
4025 request.dataset,
4026 request.version,
4027 )
4028 .await?;
4029 let subject_variable_id =
4030 require_dataset_variable_id(&variables, &request.subject_variable)?;
4031 let object_variable_id = require_dataset_variable_id(&variables, &request.object_variable)?;
4032 let relation_variable_id = request
4033 .relation_variable
4034 .as_ref()
4035 .map(|name| require_dataset_variable_id(&variables, name))
4036 .transpose()?;
4037 let link = ahri_tre_types::DatasetVersionRelationInstanceRecord {
4038 version_id: version.version_id,
4039 relation_instance_id: request.relation_instance_id,
4040 subject_variable_id,
4041 object_variable_id,
4042 relation_variable_id,
4043 transformation_id: request.transformation_id,
4044 };
4045 self.entities
4046 .link_dataset_version_relation_instance(&link)
4047 .await
4048 .map_err(|error| core_error("link dataset version relation instance", error))
4049 }
4050
4051 pub async fn get_relation_dataset_link(
4052 &self,
4053 request: GetRelationDatasetLinkRequest,
4054 ) -> Result<Option<RelationDatasetLinkReadback>, AppError> {
4055 self.entities
4056 .get_relation_instance(request.relation_instance_id)
4057 .await
4058 .map_err(|error| core_error("lookup relation instance", error))?
4059 .ok_or_else(|| {
4060 AppError::Validation(format!(
4061 "relation instance not found: {}",
4062 request.relation_instance_id
4063 ))
4064 })?;
4065 let list = self
4066 .list_relation_dataset_links(ListRelationDatasetLinksRequest {
4067 study: request.study,
4068 dataset: request.dataset,
4069 version: request.version,
4070 })
4071 .await?;
4072 Ok(list
4073 .links
4074 .into_iter()
4075 .find(|link| link.link.relation_instance_id == request.relation_instance_id))
4076 }
4077
4078 pub async fn list_relation_dataset_links(
4079 &self,
4080 request: ListRelationDatasetLinksRequest,
4081 ) -> Result<RelationDatasetLinkList, AppError> {
4082 let (_, catalog, version) = self
4083 .resolve_asset_version_by_optional_study(
4084 request.study,
4085 request.dataset,
4086 request.version,
4087 Some(ahri_tre_types::AssetType::Dataset),
4088 )
4089 .await?;
4090 let links = self
4091 .entities
4092 .list_dataset_version_relation_instances(version.version_id)
4093 .await
4094 .map_err(|error| core_error("list dataset version relation links", error))?;
4095 let mut readbacks = Vec::new();
4096 for link in links {
4097 readbacks.push(self.relation_dataset_link_readback(link).await?);
4098 }
4099 Ok(RelationDatasetLinkList {
4100 resolved_dataset: Some(catalog.asset),
4101 resolved_version: Some(version),
4102 links: readbacks,
4103 })
4104 }
4105
4106 pub async fn link_dataset_version_entities(
4107 &self,
4108 request: LinkDatasetVersionEntitiesRequest,
4109 ) -> Result<(), AppError> {
4110 self.entities
4111 .link_dataset_version_entities(&request.links)
4112 .await
4113 .map_err(|error| core_error("bulk link dataset version entities", error))
4114 }
4115
4116 pub async fn link_dataset_version_relation_instances(
4117 &self,
4118 request: LinkDatasetVersionRelationInstancesRequest,
4119 ) -> Result<(), AppError> {
4120 self.entities
4121 .link_dataset_version_relation_instances(&request.links)
4122 .await
4123 .map_err(|error| core_error("bulk link dataset version relation instances", error))
4124 }
4125
4126 pub async fn list_study_transformations(
4127 &self,
4128 request: ListStudyTransformationsRequest,
4129 ) -> Result<Vec<TransformationSummary>, AppError> {
4130 let study = self.require_study_by_name(request.study).await?;
4131 self.study_transformations(study.study_id).await
4132 }
4133
4134 pub(super) async fn study_transformations(
4135 &self,
4136 study_id: StudyId,
4137 ) -> Result<Vec<TransformationSummary>, AppError> {
4138 let assets = self
4139 .list_study_assets(ListStudyAssetsRequest { study_id })
4140 .await?;
4141 let mut study_version_ids = Vec::new();
4142 for asset in assets {
4143 let catalog = self.asset_version_catalog(asset).await?;
4144 for version in catalog.versions {
4145 study_version_ids.push(version.version_id);
4146 }
4147 }
4148 let transformations = self
4149 .transformations
4150 .list_transformations()
4151 .await
4152 .map_err(|error| core_error("list transformations", error))?;
4153 let mut summaries = BTreeMap::new();
4154 for transformation in transformations {
4155 let inputs = self
4156 .transformations
4157 .list_inputs(transformation.transformation_id)
4158 .await
4159 .map_err(|error| core_error("list transformation inputs", error))?;
4160 let outputs = self
4161 .transformations
4162 .list_outputs(transformation.transformation_id)
4163 .await
4164 .map_err(|error| core_error("list transformation outputs", error))?;
4165 if inputs
4166 .iter()
4167 .any(|link| study_version_ids.contains(&link.version_id))
4168 || outputs
4169 .iter()
4170 .any(|link| study_version_ids.contains(&link.version_id))
4171 {
4172 summaries.insert(
4173 transformation.transformation_id.0,
4174 TransformationSummary {
4175 transformation,
4176 inputs,
4177 outputs,
4178 },
4179 );
4180 }
4181 }
4182 Ok(summaries.into_values().collect())
4183 }
4184
4185 async fn require_domain_by_name(
4186 &self,
4187 domain: String,
4188 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
4189 let domain_name = parse_domain_name(domain.clone())?;
4190 self.domains
4191 .get_domain_by_name(domain_name.as_str())
4192 .await
4193 .map_err(|error| core_error("lookup domain by name", error))?
4194 .ok_or_else(|| AppError::Validation(format!("domain not found: {domain}")))
4195 }
4196
4197 async fn require_study_by_name(
4198 &self,
4199 study: String,
4200 ) -> Result<ahri_tre_types::StudyRecord, AppError> {
4201 let study_name = parse_study_name(study.clone())?;
4202 self.studies
4203 .get_study_by_name(study_name.as_str())
4204 .await
4205 .map_err(|error| core_error("lookup study by name", error))?
4206 .ok_or_else(|| AppError::Validation(format!("study not found: {study}")))
4207 }
4208
4209 async fn require_study_id(
4210 &self,
4211 study_id: ahri_tre_types::StudyId,
4212 ) -> Result<ahri_tre_types::StudyRecord, AppError> {
4213 self.studies
4214 .get_study_by_id(study_id)
4215 .await
4216 .map_err(|error| core_error("lookup study", error))?
4217 .ok_or_else(|| AppError::Validation(format!("study not found: {}", study_id.0)))
4218 }
4219
4220 async fn require_entity_by_name(
4221 &self,
4222 domain_id: ahri_tre_types::DomainId,
4223 entity: String,
4224 ) -> Result<ahri_tre_types::EntityRecord, AppError> {
4225 let entity_name = parse_entity_name(entity.clone())?;
4226 self.entities
4227 .list_entities(domain_id)
4228 .await
4229 .map_err(|error| core_error("list entities", error))?
4230 .into_iter()
4231 .find(|candidate| candidate.name == entity_name)
4232 .ok_or_else(|| AppError::Validation(format!("entity not found: {entity}")))
4233 }
4234
4235 async fn require_domain_id(
4236 &self,
4237 domain_id: ahri_tre_types::DomainId,
4238 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
4239 self.domains
4240 .get_domain_by_id(domain_id)
4241 .await
4242 .map_err(|error| core_error("lookup domain", error))?
4243 .ok_or_else(|| AppError::Validation(format!("domain not found: {}", domain_id.0)))
4244 }
4245
4246 async fn require_entity_record(
4247 &self,
4248 entity_id: ahri_tre_types::EntityId,
4249 ) -> Result<ahri_tre_types::EntityRecord, AppError> {
4250 for domain in self
4251 .domains
4252 .list_domains()
4253 .await
4254 .map_err(|error| core_error("list domains", error))?
4255 {
4256 if let Some(entity) = self
4257 .entities
4258 .list_entities(domain.domain_id)
4259 .await
4260 .map_err(|error| core_error("list entities", error))?
4261 .into_iter()
4262 .find(|entity| entity.entity_id == entity_id)
4263 {
4264 return Ok(entity);
4265 }
4266 }
4267 Err(AppError::Validation(format!(
4268 "entity not found: {}",
4269 entity_id.0
4270 )))
4271 }
4272
4273 async fn require_entity_instance_id(
4274 &self,
4275 instance_id: i64,
4276 ) -> Result<ahri_tre_types::EntityInstanceRecord, AppError> {
4277 self.entities
4278 .get_entity_instance(instance_id)
4279 .await
4280 .map_err(|error| core_error("lookup entity instance", error))?
4281 .ok_or_else(|| {
4282 AppError::Validation(format!("entity instance not found: {instance_id}"))
4283 })
4284 }
4285
4286 async fn require_relation_instance_id(
4287 &self,
4288 relation_instance_id: i64,
4289 ) -> Result<ahri_tre_types::RelationInstanceRecord, AppError> {
4290 self.entities
4291 .get_relation_instance(relation_instance_id)
4292 .await
4293 .map_err(|error| core_error("lookup relation instance", error))?
4294 .ok_or_else(|| {
4295 AppError::Validation(format!(
4296 "relation instance not found: {relation_instance_id}"
4297 ))
4298 })
4299 }
4300
4301 async fn require_entity_relation_by_name(
4302 &self,
4303 domain_id: ahri_tre_types::DomainId,
4304 relation: String,
4305 ) -> Result<ahri_tre_types::EntityRelationRecord, AppError> {
4306 let relation_name = parse_entity_relation_name(relation.clone())?;
4307 self.entities
4308 .list_entity_relations(domain_id)
4309 .await
4310 .map_err(|error| core_error("list entity relations", error))?
4311 .into_iter()
4312 .find(|candidate| candidate.name == relation_name)
4313 .ok_or_else(|| AppError::Validation(format!("entity relation not found: {relation}")))
4314 }
4315
4316 async fn resolve_study_asset_version(
4317 &self,
4318 study: String,
4319 asset: String,
4320 version: Option<String>,
4321 ) -> Result<ahri_tre_types::AssetVersionRecord, AppError> {
4322 let study = self.require_study_by_name(study).await?;
4323 let asset_name = ahri_tre_types::NcName::parse(asset.clone()).map_err(|error| {
4324 AppError::Validation(format!("invalid asset name {asset}: {error}"))
4325 })?;
4326 let catalog = self
4327 .get_asset(GetAssetRequest {
4328 asset_id: None,
4329 study_id: Some(study.study_id),
4330 asset_name: Some(asset_name),
4331 })
4332 .await?;
4333 resolve_requested_asset_version(&catalog, version.as_deref())
4334 }
4335
4336 async fn resolve_asset_version_by_optional_study(
4337 &self,
4338 study: Option<String>,
4339 asset: String,
4340 version: Option<String>,
4341 required_asset_type: Option<ahri_tre_types::AssetType>,
4342 ) -> Result<
4343 (
4344 ahri_tre_types::StudyRecord,
4345 AssetVersionCatalog,
4346 ahri_tre_types::AssetVersionRecord,
4347 ),
4348 AppError,
4349 > {
4350 let asset_name = ahri_tre_types::NcName::parse(asset.clone()).map_err(|error| {
4351 AppError::Validation(format!("invalid asset name {asset}: {error}"))
4352 })?;
4353 let mut matches = Vec::new();
4354 let studies = match study {
4355 Some(study) => vec![self.require_study_by_name(study).await?],
4356 None => self
4357 .studies
4358 .list_studies()
4359 .await
4360 .map_err(|error| core_error("list studies", error))?,
4361 };
4362 for study in studies {
4363 let maybe_catalog = self
4364 .get_asset(GetAssetRequest {
4365 asset_id: None,
4366 study_id: Some(study.study_id),
4367 asset_name: Some(asset_name.clone()),
4368 })
4369 .await;
4370 let catalog = match maybe_catalog {
4371 Ok(catalog) => catalog,
4372 Err(AppError::Validation(message)) if message.contains("asset not found") => {
4373 continue;
4374 }
4375 Err(error) => return Err(error),
4376 };
4377 if let Some(required_asset_type) = required_asset_type
4378 && catalog.asset.asset_type != required_asset_type
4379 {
4380 return Err(AppError::Validation(format!(
4381 "asset {} is not a {}",
4382 catalog.asset.name.as_str(),
4383 asset_type_label(required_asset_type)
4384 )));
4385 }
4386 matches.push((study, catalog));
4387 }
4388 let (study, catalog) = match matches.len() {
4389 0 => return Err(AppError::Validation(format!("asset not found: {asset}"))),
4390 1 => matches.into_iter().next().expect("one match should exist"),
4391 _ => {
4392 return Err(AppError::Validation(format!(
4393 "asset {asset} exists in multiple studies; supply a study selector"
4394 )));
4395 }
4396 };
4397 let resolved_version = resolve_requested_asset_version(&catalog, version.as_deref())?;
4398 Ok((study, catalog, resolved_version))
4399 }
4400
4401 async fn resolve_study_dataset_version_and_variables(
4402 &self,
4403 study: String,
4404 dataset: String,
4405 version: Option<String>,
4406 ) -> Result<
4407 (
4408 ahri_tre_types::AssetVersionRecord,
4409 Vec<DatasetVariableMetadata>,
4410 ),
4411 AppError,
4412 > {
4413 let study = self.require_study_by_name(study).await?;
4414 let dataset_name = ahri_tre_types::NcName::parse(dataset.clone()).map_err(|error| {
4415 AppError::Validation(format!("invalid dataset name {dataset}: {error}"))
4416 })?;
4417 let catalog = self
4418 .dataset_catalog_by_name(study.study_id, dataset_name)
4419 .await?;
4420 if catalog.asset.asset_type != ahri_tre_types::AssetType::Dataset {
4421 return Err(AppError::Validation(format!(
4422 "asset {} is not a dataset",
4423 catalog.asset.name.as_str()
4424 )));
4425 }
4426 let version = resolve_requested_asset_version(&catalog, version.as_deref())?;
4427 let variables = self
4428 .get_dataset_variables(GetDatasetVariablesRequest {
4429 dataset_id: version.version_id,
4430 })
4431 .await?
4432 .variables;
4433 Ok((version, variables))
4434 }
4435
4436 async fn entity_study_mapping_for_asset(
4437 &self,
4438 study_id: ahri_tre_types::StudyId,
4439 instance_id: i64,
4440 ) -> Result<Option<ahri_tre_types::StudyEntityInstanceRecord>, AppError> {
4441 Ok(self
4442 .entities
4443 .list_study_entity_instances(study_id)
4444 .await
4445 .map_err(|error| core_error("list study entity instances", error))?
4446 .into_iter()
4447 .find(|mapping| mapping.entity_instance_id == instance_id))
4448 }
4449
4450 async fn relation_study_mapping_for_asset(
4451 &self,
4452 study_id: ahri_tre_types::StudyId,
4453 relation_instance_id: i64,
4454 ) -> Result<Option<ahri_tre_types::StudyRelationInstanceRecord>, AppError> {
4455 Ok(self
4456 .entities
4457 .list_study_relation_instances(study_id)
4458 .await
4459 .map_err(|error| core_error("list study relation instances", error))?
4460 .into_iter()
4461 .find(|mapping| mapping.relation_instance_id == relation_instance_id))
4462 }
4463
4464 async fn require_entity_relation_record(
4465 &self,
4466 relation_id: ahri_tre_types::EntityRelationId,
4467 ) -> Result<ahri_tre_types::EntityRelationRecord, AppError> {
4468 for domain in self
4469 .domains
4470 .list_domains()
4471 .await
4472 .map_err(|error| core_error("list domains", error))?
4473 {
4474 if let Some(relation) = self
4475 .entities
4476 .list_entity_relations(domain.domain_id)
4477 .await
4478 .map_err(|error| core_error("list entity relations", error))?
4479 .into_iter()
4480 .find(|relation| relation.entity_relation_id == relation_id)
4481 {
4482 return Ok(relation);
4483 }
4484 }
4485 Err(AppError::Validation(format!(
4486 "entity relation not found: {}",
4487 relation_id.0
4488 )))
4489 }
4490
4491 async fn entity_asset_link_readback(
4492 &self,
4493 link: ahri_tre_types::AssetVersionEntityRecord,
4494 ) -> Result<EntityAssetLinkReadback, AppError> {
4495 let version = self
4496 .assets
4497 .get_asset_version(link.version_id)
4498 .await
4499 .map_err(|error| core_error("lookup asset version", error))?
4500 .ok_or_else(|| {
4501 AppError::Validation(format!("asset version not found: {}", link.version_id.0))
4502 })?;
4503 let asset = self
4504 .assets
4505 .get_asset_by_id(version.asset_id)
4506 .await
4507 .map_err(|error| core_error("lookup asset", error))?
4508 .ok_or_else(|| {
4509 AppError::Validation(format!("asset not found: {}", version.asset_id.0))
4510 })?;
4511 let study = self.require_study_id(asset.study_id).await?;
4512 let instance = self
4513 .require_entity_instance_id(link.entity_instance_id)
4514 .await?;
4515 let entity = self.require_entity_record(instance.entity_id).await?;
4516 let domain = self.require_domain_id(entity.domain_id).await?;
4517 let study_mapping = self
4518 .entity_study_mapping_for_asset(study.study_id, instance.instance_id)
4519 .await?;
4520 Ok(EntityAssetLinkReadback {
4521 study,
4522 asset,
4523 version,
4524 domain,
4525 entity,
4526 instance,
4527 study_mapping,
4528 link,
4529 })
4530 }
4531
4532 async fn relation_asset_link_readback(
4533 &self,
4534 link: ahri_tre_types::AssetVersionRelationInstanceRecord,
4535 ) -> Result<RelationAssetLinkReadback, AppError> {
4536 let version = self
4537 .assets
4538 .get_asset_version(link.version_id)
4539 .await
4540 .map_err(|error| core_error("lookup asset version", error))?
4541 .ok_or_else(|| {
4542 AppError::Validation(format!("asset version not found: {}", link.version_id.0))
4543 })?;
4544 let asset = self
4545 .assets
4546 .get_asset_by_id(version.asset_id)
4547 .await
4548 .map_err(|error| core_error("lookup asset", error))?
4549 .ok_or_else(|| {
4550 AppError::Validation(format!("asset not found: {}", version.asset_id.0))
4551 })?;
4552 let study = self.require_study_id(asset.study_id).await?;
4553 let instance = self
4554 .require_relation_instance_id(link.relation_instance_id)
4555 .await?;
4556 let relation = self
4557 .require_entity_relation_record(instance.entity_relation_id)
4558 .await?;
4559 let domain = self.require_domain_id(relation.domain_id).await?;
4560 let subject_entity = self
4561 .require_entity_record(relation.subject_entity_id)
4562 .await?;
4563 let object_entity = self
4564 .require_entity_record(relation.object_entity_id)
4565 .await?;
4566 let subject_instance = self
4567 .require_entity_instance_id(instance.entity_instance_id_1)
4568 .await?;
4569 let object_instance = self
4570 .require_entity_instance_id(instance.entity_instance_id_2)
4571 .await?;
4572 let study_mapping = self
4573 .relation_study_mapping_for_asset(study.study_id, instance.relation_instance_id)
4574 .await?;
4575 Ok(RelationAssetLinkReadback {
4576 study,
4577 asset,
4578 version,
4579 domain,
4580 relation,
4581 subject_entity,
4582 object_entity,
4583 subject_instance,
4584 object_instance,
4585 instance,
4586 study_mapping,
4587 link,
4588 })
4589 }
4590
4591 async fn entity_dataset_link_readback(
4592 &self,
4593 link: ahri_tre_types::DatasetVersionEntityRecord,
4594 ) -> Result<EntityDatasetLinkReadback, AppError> {
4595 let version = self
4596 .assets
4597 .get_asset_version(link.version_id)
4598 .await
4599 .map_err(|error| core_error("lookup dataset version", error))?
4600 .ok_or_else(|| {
4601 AppError::Validation(format!("dataset version not found: {}", link.version_id.0))
4602 })?;
4603 let dataset = self
4604 .assets
4605 .get_asset_by_id(version.asset_id)
4606 .await
4607 .map_err(|error| core_error("lookup dataset", error))?
4608 .ok_or_else(|| {
4609 AppError::Validation(format!("dataset asset not found: {}", version.asset_id.0))
4610 })?;
4611 if dataset.asset_type != ahri_tre_types::AssetType::Dataset {
4612 return Err(AppError::Validation(format!(
4613 "asset {} is not a dataset",
4614 dataset.name.as_str()
4615 )));
4616 }
4617 let study = self.require_study_id(dataset.study_id).await?;
4618 let instance = self
4619 .require_entity_instance_id(link.entity_instance_id)
4620 .await?;
4621 let entity = self.require_entity_record(instance.entity_id).await?;
4622 let domain = self.require_domain_id(entity.domain_id).await?;
4623 let variable = self
4624 .variables
4625 .get_variable(link.entity_variable_id)
4626 .await
4627 .map_err(|error| core_error("lookup entity dataset variable", error))?
4628 .ok_or_else(|| {
4629 AppError::Validation(format!(
4630 "dataset variable not found: {}",
4631 link.entity_variable_id.0
4632 ))
4633 })?;
4634 let study_mapping = self
4635 .entity_study_mapping_for_asset(study.study_id, instance.instance_id)
4636 .await?;
4637 Ok(EntityDatasetLinkReadback {
4638 study,
4639 dataset,
4640 version,
4641 domain,
4642 entity,
4643 instance,
4644 study_mapping,
4645 variable,
4646 link,
4647 })
4648 }
4649
4650 async fn relation_dataset_link_readback(
4651 &self,
4652 link: ahri_tre_types::DatasetVersionRelationInstanceRecord,
4653 ) -> Result<RelationDatasetLinkReadback, AppError> {
4654 let version = self
4655 .assets
4656 .get_asset_version(link.version_id)
4657 .await
4658 .map_err(|error| core_error("lookup dataset version", error))?
4659 .ok_or_else(|| {
4660 AppError::Validation(format!("dataset version not found: {}", link.version_id.0))
4661 })?;
4662 let dataset = self
4663 .assets
4664 .get_asset_by_id(version.asset_id)
4665 .await
4666 .map_err(|error| core_error("lookup dataset", error))?
4667 .ok_or_else(|| {
4668 AppError::Validation(format!("dataset asset not found: {}", version.asset_id.0))
4669 })?;
4670 if dataset.asset_type != ahri_tre_types::AssetType::Dataset {
4671 return Err(AppError::Validation(format!(
4672 "asset {} is not a dataset",
4673 dataset.name.as_str()
4674 )));
4675 }
4676 let study = self.require_study_id(dataset.study_id).await?;
4677 let instance = self
4678 .entities
4679 .get_relation_instance(link.relation_instance_id)
4680 .await
4681 .map_err(|error| core_error("lookup relation instance", error))?
4682 .ok_or_else(|| {
4683 AppError::Validation(format!(
4684 "relation instance not found: {}",
4685 link.relation_instance_id
4686 ))
4687 })?;
4688 let relation = self
4689 .require_entity_relation_record(instance.entity_relation_id)
4690 .await?;
4691 let domain = self.require_domain_id(relation.domain_id).await?;
4692 let subject_entity = self
4693 .require_entity_record(relation.subject_entity_id)
4694 .await?;
4695 let object_entity = self
4696 .require_entity_record(relation.object_entity_id)
4697 .await?;
4698 let subject_instance = self
4699 .require_entity_instance_id(instance.entity_instance_id_1)
4700 .await?;
4701 let object_instance = self
4702 .require_entity_instance_id(instance.entity_instance_id_2)
4703 .await?;
4704 let subject_variable = self
4705 .variables
4706 .get_variable(link.subject_variable_id)
4707 .await
4708 .map_err(|error| core_error("lookup relation subject dataset variable", error))?
4709 .ok_or_else(|| {
4710 AppError::Validation(format!(
4711 "dataset variable not found: {}",
4712 link.subject_variable_id.0
4713 ))
4714 })?;
4715 let object_variable = self
4716 .variables
4717 .get_variable(link.object_variable_id)
4718 .await
4719 .map_err(|error| core_error("lookup relation object dataset variable", error))?
4720 .ok_or_else(|| {
4721 AppError::Validation(format!(
4722 "dataset variable not found: {}",
4723 link.object_variable_id.0
4724 ))
4725 })?;
4726 let relation_variable = match link.relation_variable_id {
4727 Some(variable_id) => Some(
4728 self.variables
4729 .get_variable(variable_id)
4730 .await
4731 .map_err(|error| core_error("lookup relation dataset variable", error))?
4732 .ok_or_else(|| {
4733 AppError::Validation(format!(
4734 "dataset variable not found: {}",
4735 variable_id.0
4736 ))
4737 })?,
4738 ),
4739 None => None,
4740 };
4741 let study_mapping = self
4742 .relation_study_mapping_for_asset(study.study_id, instance.relation_instance_id)
4743 .await?;
4744 Ok(RelationDatasetLinkReadback {
4745 study,
4746 dataset,
4747 version,
4748 domain,
4749 relation,
4750 subject_entity,
4751 object_entity,
4752 subject_instance,
4753 object_instance,
4754 instance,
4755 study_mapping,
4756 subject_variable,
4757 object_variable,
4758 relation_variable,
4759 link,
4760 })
4761 }
4762
4763 async fn require_entity_id(
4764 &self,
4765 domain_id: ahri_tre_types::DomainId,
4766 entity_id: ahri_tre_types::EntityId,
4767 ) -> Result<ahri_tre_types::EntityRecord, AppError> {
4768 self.entities
4769 .list_entities(domain_id)
4770 .await
4771 .map_err(|error| core_error("list entities", error))?
4772 .into_iter()
4773 .find(|entity| entity.entity_id == entity_id)
4774 .ok_or_else(|| {
4775 AppError::Validation(format!(
4776 "entity {} not found in domain {}",
4777 entity_id.0, domain_id.0
4778 ))
4779 })
4780 }
4781
4782 async fn dataset_arrow_metadata(
4783 &self,
4784 dataset: &ahri_tre_types::DatasetRecord,
4785 catalog: &AssetVersionCatalog,
4786 version: &ahri_tre_types::AssetVersionRecord,
4787 ) -> Result<AhriTreArrowMetadata, AppError> {
4788 let variables = self
4789 .get_dataset_variables(GetDatasetVariablesRequest {
4790 dataset_id: dataset.dataset_id,
4791 })
4792 .await?;
4793 let mut schema = BTreeMap::new();
4794 schema.insert(
4795 metadata_keys::DATASET_ID.to_string(),
4796 dataset.dataset_id.0.to_string(),
4797 );
4798 schema.insert(
4799 metadata_keys::ASSET_ID.to_string(),
4800 catalog.asset.asset_id.0.to_string(),
4801 );
4802 schema.insert(
4803 metadata_keys::ASSET_NAME.to_string(),
4804 catalog.asset.name.as_str().to_string(),
4805 );
4806 schema.insert(
4807 metadata_keys::STUDY_ID.to_string(),
4808 catalog.asset.study_id.0.to_string(),
4809 );
4810 schema.insert(
4811 metadata_keys::VERSION_ID.to_string(),
4812 version.version_id.0.to_string(),
4813 );
4814 schema.insert(
4815 metadata_keys::VERSION.to_string(),
4816 format!("{}.{}.{}", version.major, version.minor, version.patch),
4817 );
4818 insert_optional_metadata(
4819 &mut schema,
4820 metadata_keys::DESCRIPTION,
4821 catalog.asset.description.as_deref(),
4822 );
4823 insert_optional_metadata(
4824 &mut schema,
4825 metadata_keys::VERSION_NOTE,
4826 version.version_note.as_deref(),
4827 );
4828
4829 let fields = variables
4830 .variables
4831 .into_iter()
4832 .map(dataset_variable_arrow_metadata)
4833 .collect();
4834
4835 Ok(AhriTreArrowMetadata { schema, fields })
4836 }
4837
4838 pub async fn register_domain(
4839 &self,
4840 request: RegisterDomainRequest,
4841 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
4842 let extraction = ahri_tre_core::extract_inline_tags(request.domain.description.as_deref());
4843 let mut requested_domain = request.domain;
4844 requested_domain.description = extraction.cleaned_text;
4845 match self
4846 .domains
4847 .get_domain_by_name(requested_domain.name.as_str())
4848 .await
4849 .map_err(|error| core_error("lookup domain by name", error))?
4850 {
4851 Some(domain) => Ok(domain),
4852 None => self
4853 .registrations
4854 .save_domain_registration(&requested_domain, &extraction.tags)
4855 .await
4856 .map_err(|error| core_error("save Domain registration", error)),
4857 }
4858 }
4859
4860 pub async fn list_domains(
4861 &self,
4862 _request: ListDomainsRequest,
4863 ) -> Result<DomainListResult, AppError> {
4864 let domains = self
4865 .domains
4866 .list_domains()
4867 .await
4868 .map_err(|error| core_error("list domains", error))?;
4869 Ok(DomainListResult { domains })
4870 }
4871
4872 pub async fn get_domain(
4873 &self,
4874 request: GetDomainRequest,
4875 ) -> Result<Option<ahri_tre_types::DomainRecord>, AppError> {
4876 self.resolve_domain_selector_optional(request.domain).await
4877 }
4878
4879 pub async fn add_domain(&self, request: AddDomainRequest) -> Result<DomainAddResult, AppError> {
4880 let name = parse_domain_name(request.name)?;
4881 let extraction = ahri_tre_core::extract_inline_tags(request.description.as_deref());
4882 let requested_domain = ahri_tre_types::NewDomainRecord {
4883 name,
4884 description: extraction.cleaned_text,
4885 uri: request.uri,
4886 };
4887 let domain = self
4888 .register_normalized_domain(requested_domain, extraction.tags)
4889 .await?;
4890 Ok(DomainAddResult { domain })
4891 }
4892
4893 pub async fn register_new_domain(
4894 &self,
4895 request: RegisterNewDomainRequest,
4896 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
4897 if let Some(existing) = self
4898 .resolve_domain_selector_optional(DomainSelector::Name {
4899 name: request.domain.name.as_str().into(),
4900 })
4901 .await?
4902 {
4903 return Ok(existing);
4904 }
4905 let extraction = ahri_tre_core::extract_inline_tags(request.domain.description.as_deref());
4906 let mut requested_domain = request.domain;
4907 requested_domain.description = extraction.cleaned_text;
4908 self.register_normalized_domain(requested_domain, extraction.tags)
4909 .await
4910 }
4911
4912 async fn register_normalized_domain(
4913 &self,
4914 requested_domain: ahri_tre_types::NewDomainRecord,
4915 tags: Vec<String>,
4916 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
4917 let domains = self
4918 .domains
4919 .list_domains_by_name(requested_domain.name.as_str())
4920 .await
4921 .map_err(|error| core_error("lookup Domain registration", error))?;
4922 let mut same_name = domains.into_iter().peekable();
4923 let name_exists = same_name.peek().is_some();
4924 if let Some(existing) = same_name.find(|domain| domain.uri == requested_domain.uri) {
4925 if existing.description != requested_domain.description {
4926 return Err(AppError::Conflict(
4927 "Domain registration conflicts with existing catalogue metadata".into(),
4928 ));
4929 }
4930 return Ok(existing);
4931 }
4932 if requested_domain.uri.is_none() && name_exists {
4933 return Err(AppError::Conflict(
4934 "An explicit URI is required for a same-named Domain".into(),
4935 ));
4936 }
4937 self.registrations
4938 .create_domain_registration(&requested_domain, &tags)
4939 .await
4940 .map_err(|error| core_error("create Domain registration", error))
4941 }
4942
4943 pub async fn list_studies(
4944 &self,
4945 _request: ListStudiesRequest,
4946 ) -> Result<StudyListResult, AppError> {
4947 let studies = self
4948 .studies
4949 .list_studies()
4950 .await
4951 .map_err(|error| core_error("list studies", error))?;
4952 let mut registrations = Vec::with_capacity(studies.len());
4953 for study in studies {
4954 let domains = self
4955 .study_domains
4956 .list_domains_for_study(study.study_id)
4957 .await
4958 .map_err(|error| core_error("list study domains", error))?;
4959 registrations.push(self.study_registration_with_tags(study, domains).await?);
4960 }
4961 Ok(StudyListResult {
4962 studies: registrations,
4963 })
4964 }
4965
4966 async fn study_registration_with_tags(
4967 &self,
4968 study: StudyRecord,
4969 domains: Vec<DomainRecord>,
4970 ) -> Result<StudyRegistration, AppError> {
4971 let mut tags = self
4972 .tags
4973 .list_tags_for_target(TagAttachmentTarget::Study(study.study_id))
4974 .await
4975 .map_err(|error| core_error("list study tags", error))?;
4976 tags.truncate(MAX_SUMMARY_TAGS);
4977 Ok(StudyRegistration {
4978 study,
4979 domains,
4980 tags,
4981 })
4982 }
4983
4984 pub async fn search_studies(
4985 &self,
4986 request: SearchStudiesRequest,
4987 ) -> Result<StudySearchResult, AppError> {
4988 let query = app_study_search_query(request)?;
4989 let page = self
4990 .studies
4991 .search_studies(&query)
4992 .await
4993 .map_err(|error| core_error("search studies", error))?;
4994 let mut studies = Vec::with_capacity(page.items.len());
4995 for study in page.items {
4996 let domains = self
4997 .study_domains
4998 .list_domains_for_study(study.study_id)
4999 .await
5000 .map_err(|error| core_error("list study domains", error))?;
5001 let mut tags = self
5002 .tags
5003 .list_tags_for_target(TagAttachmentTarget::Study(study.study_id))
5004 .await
5005 .map_err(|error| core_error("list study tags", error))?;
5006 tags.truncate(MAX_SUMMARY_TAGS);
5007 studies.push(StudySearchEntry {
5008 study,
5009 domains,
5010 tags,
5011 });
5012 }
5013 Ok(StudySearchResult {
5014 studies,
5015 next_cursor: page.next_cursor,
5016 })
5017 }
5018
5019 pub async fn search_datasets(
5020 &self,
5021 request: SearchDatasetsRequest,
5022 ) -> Result<DatasetSearchResult, AppError> {
5023 let query = app_dataset_search_query(request)?;
5024 let page = self
5025 .assets
5026 .search_datasets(&query)
5027 .await
5028 .map_err(|error| core_error("search datasets", error))?;
5029 let mut datasets = Vec::with_capacity(page.items.len());
5030 for entry in page.items {
5031 let study = self
5032 .studies
5033 .get_study_by_id(entry.study_id)
5034 .await
5035 .map_err(|error| core_error("lookup search dataset study", error))?
5036 .ok_or_else(|| {
5037 AppError::Infrastructure(format!(
5038 "search dataset study not found: {}",
5039 entry.study_id.0
5040 ))
5041 })?;
5042 let variable_count = self
5043 .variables
5044 .list_dataset_variables(entry.latest_version.version_id)
5045 .await
5046 .map_err(|error| core_error("list search dataset variables", error))
5047 .ok()
5048 .and_then(|variables| u64::try_from(variables.len()).ok());
5049 let mut tags = self
5050 .tags
5051 .list_tags_for_target(TagAttachmentTarget::AssetVersion(
5052 entry.latest_version.version_id,
5053 ))
5054 .await
5055 .map_err(|error| core_error("list search dataset tags", error))?;
5056 tags.truncate(MAX_SUMMARY_TAGS);
5057 datasets.push(DatasetSearchEntry {
5058 study,
5059 asset: entry.asset,
5060 latest_version: entry.latest_version,
5061 variable_count,
5062 tags,
5063 });
5064 }
5065 Ok(DatasetSearchResult {
5066 datasets,
5067 next_cursor: page.next_cursor,
5068 })
5069 }
5070
5071 pub async fn search_datafiles(
5072 &self,
5073 request: SearchDataFilesRequest,
5074 ) -> Result<DataFileSearchResult, AppError> {
5075 let query = app_datafile_search_query(request)?;
5076 let page = self
5077 .assets
5078 .search_datafiles(&query)
5079 .await
5080 .map_err(|error| core_error("search datafiles", error))?;
5081 let mut datafiles = Vec::with_capacity(page.items.len());
5082 for entry in page.items {
5083 let study = self
5084 .studies
5085 .get_study_by_id(entry.study_id)
5086 .await
5087 .map_err(|error| core_error("lookup search datafile study", error))?
5088 .ok_or_else(|| {
5089 AppError::Infrastructure(format!(
5090 "search datafile study not found: {}",
5091 entry.study_id.0
5092 ))
5093 })?;
5094 let mut tags = self
5095 .tags
5096 .list_tags_for_target(TagAttachmentTarget::AssetVersion(
5097 entry.latest_version.version_id,
5098 ))
5099 .await
5100 .map_err(|error| core_error("list search datafile tags", error))?;
5101 tags.truncate(MAX_SUMMARY_TAGS);
5102 datafiles.push(DataFileSearchEntry {
5103 study,
5104 asset: entry.asset,
5105 latest_version: entry.latest_version,
5106 datafile: entry.datafile,
5107 tags,
5108 });
5109 }
5110 Ok(DataFileSearchResult {
5111 datafiles,
5112 next_cursor: page.next_cursor,
5113 })
5114 }
5115
5116 pub async fn get_study(
5117 &self,
5118 request: GetStudyRequest,
5119 ) -> Result<Option<StudyRegistration>, AppError> {
5120 let Some(study) = self.resolve_study_selector_optional(request.study).await? else {
5121 return Ok(None);
5122 };
5123 let domains = self
5124 .study_domains
5125 .list_domains_for_study(study.study_id)
5126 .await
5127 .map_err(|error| core_error("list study domains", error))?;
5128 Ok(Some(
5129 self.study_registration_with_tags(study, domains).await?,
5130 ))
5131 }
5132
5133 pub async fn add_study(&self, request: AddStudyRequest) -> Result<StudyAddResult, AppError> {
5134 let study_name = parse_study_name(request.name)?;
5135 let domain = self
5136 .resolve_domain_selector(request.domain, "study domain")
5137 .await?;
5138 let study_type_id = parse_study_type_id(&request.study_type)?;
5139 if let Some(existing) = self
5140 .studies
5141 .get_study_by_name(study_name.as_str())
5142 .await
5143 .map_err(|error| core_error("lookup study registration conflict", error))?
5144 {
5145 let domains = self
5146 .study_domains
5147 .list_domains_for_study(existing.study_id)
5148 .await
5149 .map_err(|error| core_error("list study registration domains", error))?;
5150 if existing.external_id.as_deref() != Some(request.external_id.as_str())
5151 || existing.study_type_id != Some(study_type_id)
5152 || !domains
5153 .iter()
5154 .any(|existing_domain| existing_domain.domain_id == domain.domain_id)
5155 {
5156 return Err(AppError::Conflict(
5157 "study registration conflicts with existing catalogue metadata".to_string(),
5158 ));
5159 }
5160 }
5161 let registration = self
5162 .register_new_study(RegisterNewStudyRequest {
5163 study: ahri_tre_types::NewStudyRecord {
5164 name: study_name,
5165 description: request.description,
5166 documentation: None,
5167 agent_instructions: None,
5168 external_id: Some(request.external_id),
5169 study_type_id: Some(study_type_id),
5170 date_created: None,
5171 created_by: None,
5172 },
5173 domain_ids: vec![domain.domain_id],
5174 })
5175 .await?;
5176 Ok(StudyAddResult { registration })
5177 }
5178
5179 pub async fn add_study_domain(
5180 &self,
5181 request: AddStudyDomainRequest,
5182 ) -> Result<StudyRegistration, AppError> {
5183 let study = self
5184 .resolve_study_selector(request.study, "study-domain link study")
5185 .await?;
5186 let domain = self
5187 .resolve_domain_selector(request.domain, "study-domain link domain")
5188 .await?;
5189
5190 self.study_domains
5191 .link_study_domain(study.study_id, domain.domain_id)
5192 .await
5193 .map_err(|error| core_error("link study domain", error))?;
5194
5195 self.get_study_registration(GetStudyRegistrationRequest {
5196 study_id: Some(study.study_id),
5197 study_name: None,
5198 })
5199 .await
5200 }
5201
5202 async fn resolve_domain_selector(
5203 &self,
5204 selector: DomainSelector,
5205 context: &str,
5206 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
5207 let label = domain_selector_label(&selector);
5208 self.resolve_domain_selector_optional(selector)
5209 .await?
5210 .ok_or_else(|| AppError::NotFound(format!("{context} not found: {label}")))
5211 }
5212
5213 async fn resolve_domain_selector_optional(
5214 &self,
5215 selector: DomainSelector,
5216 ) -> Result<Option<ahri_tre_types::DomainRecord>, AppError> {
5217 match selector {
5218 DomainSelector::Id { domain_id } => self
5219 .domains
5220 .get_domain_by_id(domain_id)
5221 .await
5222 .map_err(|error| core_error("lookup domain by id", error)),
5223 DomainSelector::Name { name } => {
5224 let name = parse_domain_name(name)?;
5225 let domains = self
5226 .domains
5227 .list_domains_by_name(name.as_str())
5228 .await
5229 .map_err(|error| core_error("lookup domain by name", error))?;
5230 let mut matches = domains.into_iter();
5231 let first = matches.next();
5232 if matches.next().is_some() {
5233 return Err(AppError::Conflict(
5234 "Domain name is ambiguous; use its reference".into(),
5235 ));
5236 }
5237 Ok(first)
5238 }
5239 }
5240 }
5241
5242 async fn resolve_study_selector(
5243 &self,
5244 selector: StudySelector,
5245 context: &str,
5246 ) -> Result<ahri_tre_types::StudyRecord, AppError> {
5247 let label = study_selector_label(&selector);
5248 self.resolve_study_selector_optional(selector)
5249 .await?
5250 .ok_or_else(|| AppError::NotFound(format!("{context} not found: {label}")))
5251 }
5252
5253 async fn resolve_study_selector_optional(
5254 &self,
5255 selector: StudySelector,
5256 ) -> Result<Option<ahri_tre_types::StudyRecord>, AppError> {
5257 match selector {
5258 StudySelector::Id { study_id } => self
5259 .studies
5260 .get_study_by_id(study_id)
5261 .await
5262 .map_err(|error| core_error("lookup study by id", error)),
5263 StudySelector::Name { domain, name } => {
5264 let study_name = parse_study_name(name)?;
5265 let Some(study) = self
5266 .studies
5267 .get_study_by_name(study_name.as_str())
5268 .await
5269 .map_err(|error| core_error("lookup study by name", error))?
5270 else {
5271 return Ok(None);
5272 };
5273 if let Some(domain_selector) = domain {
5274 let domain = self
5275 .resolve_domain_selector(domain_selector, "study selector domain")
5276 .await?;
5277 self.ensure_study_domain(study.study_id, domain.domain_id, "study selector")
5278 .await
5279 .map_err(|error| match error {
5280 AppError::Validation(_) => AppError::Conflict(
5281 "Study selector Domain disagrees with Study membership".into(),
5282 ),
5283 other => other,
5284 })?;
5285 }
5286 Ok(Some(study))
5287 }
5288 }
5289 }
5290
5291 async fn resolve_variable_selector_optional(
5292 &self,
5293 selector: VariableSelector,
5294 ) -> Result<Option<ahri_tre_types::VariableRecord>, AppError> {
5295 match selector {
5296 VariableSelector::Id { variable_id } => self
5297 .variables
5298 .get_variable(variable_id)
5299 .await
5300 .map_err(|error| core_error("lookup variable by id", error)),
5301 VariableSelector::Name { domain, name } => {
5302 let domain = self
5303 .resolve_domain_selector(domain, "variable domain")
5304 .await?;
5305 let variable_name = parse_variable_name(name)?;
5306 Ok(self
5307 .variables
5308 .list_domain_variables(domain.domain_id)
5309 .await
5310 .map_err(|error| core_error("list domain variables", error))?
5311 .into_iter()
5312 .find(|variable| variable.name == variable_name))
5313 }
5314 }
5315 }
5316
5317 async fn resolve_vocabulary_selector_optional(
5318 &self,
5319 selector: VocabularySelector,
5320 ) -> Result<Option<ahri_tre_types::VocabularyRecord>, AppError> {
5321 match selector {
5322 VocabularySelector::Id { vocabulary_id } => self
5323 .vocabularies
5324 .get_vocabulary(vocabulary_id)
5325 .await
5326 .map_err(|error| core_error("lookup vocabulary by id", error)),
5327 VocabularySelector::Name { domain, name } => {
5328 let domain = self
5329 .resolve_domain_selector(domain, "vocabulary domain")
5330 .await?;
5331 let vocabulary_name = parse_vocabulary_name(name)?;
5332 Ok(self
5333 .vocabularies
5334 .list_vocabularies(domain.domain_id)
5335 .await
5336 .map_err(|error| core_error("list vocabularies", error))?
5337 .into_iter()
5338 .find(|vocabulary| vocabulary.name == vocabulary_name))
5339 }
5340 }
5341 }
5342
5343 async fn resolve_entity_selector_optional(
5344 &self,
5345 selector: EntitySelector,
5346 ) -> Result<Option<ahri_tre_types::EntityRecord>, AppError> {
5347 match selector {
5348 EntitySelector::Id { entity_id } => match self.require_entity_record(entity_id).await {
5349 Ok(entity) => Ok(Some(entity)),
5350 Err(AppError::Validation(message))
5351 if message == format!("entity not found: {}", entity_id.0) =>
5352 {
5353 Ok(None)
5354 }
5355 Err(error) => Err(error),
5356 },
5357 EntitySelector::Name { domain, name } => {
5358 let domain = self
5359 .resolve_domain_selector(domain, "entity domain")
5360 .await?;
5361 let entity_name = parse_entity_name(name)?;
5362 Ok(self
5363 .entities
5364 .list_entities(domain.domain_id)
5365 .await
5366 .map_err(|error| core_error("list entities", error))?
5367 .into_iter()
5368 .find(|entity| entity.name == entity_name))
5369 }
5370 }
5371 }
5372
5373 async fn resolve_entity_relation_selector_optional(
5374 &self,
5375 selector: EntityRelationSelector,
5376 ) -> Result<Option<ahri_tre_types::EntityRelationRecord>, AppError> {
5377 match selector {
5378 EntityRelationSelector::Id { entity_relation_id } => {
5379 match self
5380 .require_entity_relation_record(entity_relation_id)
5381 .await
5382 {
5383 Ok(relation) => Ok(Some(relation)),
5384 Err(AppError::Validation(message))
5385 if message
5386 == format!("entity relation not found: {}", entity_relation_id.0) =>
5387 {
5388 Ok(None)
5389 }
5390 Err(error) => Err(error),
5391 }
5392 }
5393 EntityRelationSelector::Name { domain, name } => {
5394 let domain = self
5395 .resolve_domain_selector(domain, "entity relation domain")
5396 .await?;
5397 let relation_name = parse_entity_relation_name(name)?;
5398 Ok(self
5399 .entities
5400 .list_entity_relations(domain.domain_id)
5401 .await
5402 .map_err(|error| core_error("list entity relations", error))?
5403 .into_iter()
5404 .find(|relation| relation.name == relation_name))
5405 }
5406 }
5407 }
5408
5409 pub async fn register_study(
5410 &self,
5411 request: RegisterStudyRequest,
5412 ) -> Result<StudyRegistration, AppError> {
5413 if request.domain_ids.is_empty() {
5414 return Err(AppError::Validation(
5415 "study registration requires at least one domain".to_string(),
5416 ));
5417 }
5418
5419 for domain_id in &request.domain_ids {
5420 self.domains
5421 .get_domain_by_id(*domain_id)
5422 .await
5423 .map_err(|error| core_error("lookup study domain", error))?
5424 .ok_or_else(|| {
5425 AppError::NotFound(format!("study domain not found: {}", domain_id.0))
5426 })?;
5427 }
5428
5429 let extraction = ahri_tre_core::extract_inline_tags(request.study.description.as_deref());
5430 let mut requested_study = request.study;
5431 requested_study.description = extraction.cleaned_text;
5432
5433 let study = self
5434 .registrations
5435 .save_study_registration(&requested_study, &request.domain_ids, &extraction.tags)
5436 .await
5437 .map_err(|error| core_error("save Study registration", error))?;
5438
5439 self.get_study_registration(GetStudyRegistrationRequest {
5440 study_id: Some(study.study_id),
5441 study_name: None,
5442 })
5443 .await
5444 }
5445
5446 pub async fn register_new_study(
5447 &self,
5448 request: RegisterNewStudyRequest,
5449 ) -> Result<StudyRegistration, AppError> {
5450 if request.domain_ids.is_empty() {
5451 return Err(AppError::Validation(
5452 "study registration requires at least one domain".to_string(),
5453 ));
5454 }
5455
5456 let mut domains = Vec::with_capacity(request.domain_ids.len());
5457 for domain_id in &request.domain_ids {
5458 let domain = self
5459 .domains
5460 .get_domain_by_id(*domain_id)
5461 .await
5462 .map_err(|error| core_error("lookup study domain", error))?
5463 .ok_or_else(|| {
5464 AppError::NotFound(format!("study domain not found: {}", domain_id.0))
5465 })?;
5466 domains.push(domain);
5467 }
5468
5469 let extraction = ahri_tre_core::extract_inline_tags(request.study.description.as_deref());
5470 let mut requested_study = request.study;
5471 requested_study.description = extraction.cleaned_text;
5472
5473 match self
5474 .studies
5475 .get_study_by_name(requested_study.name.as_str())
5476 .await
5477 .map_err(|error| core_error("lookup study by name", error))?
5478 {
5479 Some(study) => {
5480 self.get_study_registration(GetStudyRegistrationRequest {
5481 study_id: Some(study.study_id),
5482 study_name: None,
5483 })
5484 .await
5485 }
5486 None => {
5487 let study = self
5488 .registrations
5489 .create_study_registration(
5490 &requested_study,
5491 &request.domain_ids,
5492 &extraction.tags,
5493 )
5494 .await
5495 .map_err(|error| core_error("create Study registration", error))?;
5496 self.study_registration_with_tags(study, domains).await
5497 }
5498 }
5499 }
5500
5501 pub async fn get_study_registration(
5502 &self,
5503 request: GetStudyRegistrationRequest,
5504 ) -> Result<StudyRegistration, AppError> {
5505 let study = match (request.study_id, request.study_name) {
5506 (Some(study_id), None) => self
5507 .studies
5508 .get_study_by_id(study_id)
5509 .await
5510 .map_err(|error| core_error("lookup study by id", error))?
5511 .ok_or_else(|| AppError::NotFound(format!("study: {}", study_id.0)))?,
5512 (None, Some(study_name)) => self
5513 .studies
5514 .get_study_by_name(study_name.as_str())
5515 .await
5516 .map_err(|error| core_error("lookup study by name", error))?
5517 .ok_or_else(|| AppError::NotFound(format!("study: {}", study_name.as_str())))?,
5518 (None, None) => {
5519 return Err(AppError::Validation(
5520 "study registration lookup requires a study_id or study_name".to_string(),
5521 ));
5522 }
5523 (Some(_), Some(_)) => {
5524 return Err(AppError::Validation(
5525 "study registration lookup accepts either study_id or study_name, not both"
5526 .to_string(),
5527 ));
5528 }
5529 };
5530
5531 let domains = self
5532 .study_domains
5533 .list_domains_for_study(study.study_id)
5534 .await
5535 .map_err(|error| core_error("list study domains", error))?;
5536 self.study_registration_with_tags(study, domains).await
5537 }
5538
5539 pub async fn list_study_access_grants(
5540 &self,
5541 request: ListStudyAccessGrantsRequest,
5542 ) -> Result<StudyAccessGrantList, AppError> {
5543 let grants = self
5544 .study_governance
5545 .list_access_grants(request.study_id)
5546 .await
5547 .map_err(|error| core_error("list study access grants", error))?;
5548
5549 Ok(StudyAccessGrantList {
5550 study_id: request.study_id,
5551 grants,
5552 })
5553 }
5554
5555 pub async fn grant_study_access(
5556 &self,
5557 request: GrantStudyAccessRequest,
5558 ) -> Result<StudyAccessGrant, AppError> {
5559 let target_user_id = request.target_user_id.trim();
5560 if target_user_id.is_empty() {
5561 return Err(AppError::Validation(
5562 "study access target user must not be empty".to_string(),
5563 ));
5564 }
5565
5566 let grant = self
5567 .study_governance
5568 .grant_study_access(request.study_id, target_user_id)
5569 .await
5570 .map_err(|error| core_error("grant study access", error))?;
5571
5572 Ok(StudyAccessGrant { grant })
5573 }
5574
5575 pub async fn revoke_study_access(
5576 &self,
5577 request: RevokeStudyAccessRequest,
5578 ) -> Result<StudyAccessRevocation, AppError> {
5579 let target_user_id = request.target_user_id.trim();
5580 if target_user_id.is_empty() {
5581 return Err(AppError::Validation(
5582 "study access target user must not be empty".to_string(),
5583 ));
5584 }
5585
5586 self.study_governance
5587 .revoke_study_access(request.study_id, target_user_id)
5588 .await
5589 .map_err(|error| core_error("revoke study access", error))?;
5590
5591 Ok(StudyAccessRevocation {
5592 study_id: request.study_id,
5593 target_user_id: target_user_id.to_string(),
5594 })
5595 }
5596
5597 pub async fn list_study_custodians(
5598 &self,
5599 request: ListStudyCustodiansRequest,
5600 ) -> Result<StudyCustodianList, AppError> {
5601 let custodians = self
5602 .study_governance
5603 .list_custodians(request.study_id)
5604 .await
5605 .map_err(|error| governance_core_error("list study custodians", error))?;
5606
5607 Ok(StudyCustodianList {
5608 study_id: request.study_id,
5609 custodians,
5610 })
5611 }
5612
5613 pub async fn add_delegate_custodian(
5614 &self,
5615 request: AddDelegateCustodianRequest,
5616 ) -> Result<StudyCustodian, AppError> {
5617 let target_user_id = request.target_user_id.trim();
5618 if target_user_id.is_empty() {
5619 return Err(AppError::Validation(
5620 "delegate custodian target user must not be empty".to_string(),
5621 ));
5622 }
5623
5624 let custodian = self
5625 .study_governance
5626 .add_delegate_custodian(request.study_id, target_user_id)
5627 .await
5628 .map_err(|error| governance_core_error("add delegate custodian", error))?;
5629
5630 Ok(StudyCustodian { custodian })
5631 }
5632
5633 pub async fn transfer_primary_custodianship(
5634 &self,
5635 request: TransferPrimaryCustodianshipRequest,
5636 ) -> Result<StudyPrimaryCustodianshipTransfer, AppError> {
5637 let target_user_id = request.target_user_id.trim();
5638 if target_user_id.is_empty() {
5639 return Err(AppError::Validation(
5640 "primary custodian target user must not be empty".to_string(),
5641 ));
5642 }
5643
5644 let primary_custodian = self
5645 .study_governance
5646 .transfer_primary_custodianship(request.study_id, target_user_id)
5647 .await
5648 .map_err(|error| governance_core_error("transfer primary custodianship", error))?;
5649
5650 Ok(StudyPrimaryCustodianshipTransfer { primary_custodian })
5651 }
5652
5653 pub async fn remove_delegate_custodian(
5654 &self,
5655 request: RemoveDelegateCustodianRequest,
5656 ) -> Result<StudyDelegateCustodianRemoval, AppError> {
5657 let target_user_id = request.target_user_id.trim();
5658 if target_user_id.is_empty() {
5659 return Err(AppError::Validation(
5660 "delegate custodian target user must not be empty".to_string(),
5661 ));
5662 }
5663
5664 self.study_governance
5665 .remove_delegate_custodian(request.study_id, target_user_id)
5666 .await
5667 .map_err(|error| governance_core_error("remove delegate custodian", error))?;
5668
5669 Ok(StudyDelegateCustodianRemoval {
5670 study_id: request.study_id,
5671 target_user_id: target_user_id.to_string(),
5672 })
5673 }
5674
5675 pub async fn list_study_duo_restrictions(
5676 &self,
5677 request: ListStudyDuoRestrictionsRequest,
5678 ) -> Result<StudyDuoRestrictionList, AppError> {
5679 let restrictions = self
5680 .study_governance
5681 .list_study_duo_restrictions(request.study_id)
5682 .await
5683 .map_err(|error| governance_core_error("list study DUO restrictions", error))?;
5684
5685 Ok(StudyDuoRestrictionList {
5686 study_id: request.study_id,
5687 restrictions,
5688 })
5689 }
5690
5691 pub async fn replace_study_duo_restrictions(
5692 &self,
5693 request: ReplaceStudyDuoRestrictionsRequest,
5694 ) -> Result<StudyDuoRestrictionReplacement, AppError> {
5695 let mut restrictions = Vec::with_capacity(request.restrictions.len());
5696 for restriction in request.restrictions {
5697 restrictions.push(
5698 restriction
5699 .into_new_restriction(request.study_id)
5700 .map_err(AppError::Validation)?,
5701 );
5702 }
5703
5704 let restrictions = self
5705 .study_governance
5706 .replace_study_duo_restrictions(request.study_id, &restrictions)
5707 .await
5708 .map_err(|error| governance_core_error("replace study DUO restrictions", error))?;
5709
5710 Ok(StudyDuoRestrictionReplacement {
5711 study_id: request.study_id,
5712 restrictions,
5713 })
5714 }
5715
5716 pub async fn list_asset_version_duo_restrictions(
5717 &self,
5718 request: ListAssetVersionDuoRestrictionsRequest,
5719 ) -> Result<AssetVersionDuoRestrictionList, AppError> {
5720 let restrictions = self
5721 .assets
5722 .list_asset_version_duo_restrictions(request.version_id)
5723 .await
5724 .map_err(|error| core_error("list asset version DUO restrictions", error))?;
5725
5726 Ok(AssetVersionDuoRestrictionList {
5727 version_id: request.version_id,
5728 restrictions,
5729 })
5730 }
5731
5732 pub async fn list_effective_asset_version_duo_restrictions(
5733 &self,
5734 request: ListEffectiveAssetVersionDuoRestrictionsRequest,
5735 ) -> Result<EffectiveAssetVersionDuoRestrictionList, AppError> {
5736 let explicit = self
5737 .assets
5738 .list_asset_version_duo_restrictions(request.version_id)
5739 .await
5740 .map_err(|error| core_error("list asset version DUO restrictions", error))?;
5741
5742 let restrictions = if explicit.is_empty() {
5743 self.study_governance
5744 .list_study_duo_restrictions(request.study_id)
5745 .await
5746 .map_err(|error| governance_core_error("list study DUO restrictions", error))?
5747 .into_iter()
5748 .map(|restriction| EffectiveAssetVersionDuoRestriction {
5749 source: AssetDuoRestrictionSource::StudyDefault,
5750 duo_id: restriction.duo_id,
5751 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace,
5752 qualifier_ontology_id: restriction.qualifier_ontology_id,
5753 qualifier_text: restriction.qualifier_text,
5754 qualifier_date: restriction.qualifier_date,
5755 qualifier_integer: restriction.qualifier_integer,
5756 })
5757 .collect()
5758 } else {
5759 explicit
5760 .into_iter()
5761 .map(|restriction| EffectiveAssetVersionDuoRestriction {
5762 source: AssetDuoRestrictionSource::AssetVersion,
5763 duo_id: restriction.duo_id,
5764 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace,
5765 qualifier_ontology_id: restriction.qualifier_ontology_id,
5766 qualifier_text: restriction.qualifier_text,
5767 qualifier_date: restriction.qualifier_date,
5768 qualifier_integer: restriction.qualifier_integer,
5769 })
5770 .collect()
5771 };
5772
5773 Ok(EffectiveAssetVersionDuoRestrictionList {
5774 study_id: request.study_id,
5775 version_id: request.version_id,
5776 restrictions,
5777 })
5778 }
5779
5780 pub async fn replace_asset_version_duo_restrictions(
5781 &self,
5782 request: ReplaceAssetVersionDuoRestrictionsRequest,
5783 ) -> Result<AssetVersionDuoRestrictionReplacement, AppError> {
5784 if request.restrictions.is_empty() {
5785 return Err(AppError::Validation(
5786 "asset duo replace requires at least one restriction; use asset duo clear to remove explicit overrides".to_string(),
5787 ));
5788 }
5789
5790 let mut restrictions = Vec::with_capacity(request.restrictions.len());
5791 for restriction in request.restrictions {
5792 restrictions.push(
5793 restriction
5794 .into_new_restriction(request.version_id)
5795 .map_err(AppError::Validation)?,
5796 );
5797 }
5798
5799 let restrictions = self
5800 .assets
5801 .replace_asset_version_duo_restrictions(request.version_id, &restrictions)
5802 .await
5803 .map_err(|error| core_error("replace asset version DUO restrictions", error))?;
5804
5805 Ok(AssetVersionDuoRestrictionReplacement {
5806 version_id: request.version_id,
5807 restrictions,
5808 })
5809 }
5810
5811 pub async fn clear_asset_version_duo_restrictions(
5812 &self,
5813 request: ClearAssetVersionDuoRestrictionsRequest,
5814 ) -> Result<AssetVersionDuoRestrictionClear, AppError> {
5815 let restrictions = self
5816 .assets
5817 .clear_asset_version_duo_restrictions(request.version_id)
5818 .await
5819 .map_err(|error| core_error("clear asset version DUO restrictions", error))?;
5820
5821 Ok(AssetVersionDuoRestrictionClear {
5822 version_id: request.version_id,
5823 restrictions,
5824 })
5825 }
5826
5827 pub async fn create_asset(
5828 &self,
5829 request: CreateAssetRequest,
5830 ) -> Result<AssetVersionCatalog, AppError> {
5831 let extraction = ahri_tre_core::extract_inline_tags(request.asset.description.as_deref());
5832 let mut requested_asset = request.asset;
5833 requested_asset.description = extraction.cleaned_text;
5834 self.studies
5835 .get_study_by_id(requested_asset.study_id)
5836 .await
5837 .map_err(|error| core_error("lookup asset study", error))?
5838 .ok_or_else(|| {
5839 AppError::Validation(format!(
5840 "asset study not found: {}",
5841 requested_asset.study_id.0
5842 ))
5843 })?;
5844
5845 if let Some(existing) = self
5846 .assets
5847 .get_asset_by_name(requested_asset.study_id, requested_asset.name.as_str())
5848 .await
5849 .map_err(|error| core_error("lookup asset by name", error))?
5850 {
5851 return Err(AppError::Validation(format!(
5852 "asset already exists in study {}: {} ({})",
5853 requested_asset.study_id.0,
5854 requested_asset.name.as_str(),
5855 existing.asset_id.0
5856 )));
5857 }
5858
5859 let asset = self
5860 .assets
5861 .save_asset(&requested_asset)
5862 .await
5863 .map_err(|error| core_error("save asset", error))?;
5864 self.attach_tags(TagAttachmentTarget::Asset(asset.asset_id), extraction.tags)
5865 .await?;
5866
5867 self.get_asset(GetAssetRequest {
5868 asset_id: Some(asset.asset_id),
5869 study_id: None,
5870 asset_name: None,
5871 })
5872 .await
5873 }
5874
5875 pub async fn create_new_asset(
5876 &self,
5877 request: CreateNewAssetRequest,
5878 ) -> Result<AssetVersionCatalog, AppError> {
5879 let extraction = ahri_tre_core::extract_inline_tags(request.asset.description.as_deref());
5880 let mut requested_asset = request.asset;
5881 requested_asset.description = extraction.cleaned_text;
5882 self.studies
5883 .get_study_by_id(requested_asset.study_id)
5884 .await
5885 .map_err(|error| core_error("lookup asset study", error))?
5886 .ok_or_else(|| {
5887 AppError::Validation(format!(
5888 "asset study not found: {}",
5889 requested_asset.study_id.0
5890 ))
5891 })?;
5892
5893 if let Some(existing) = self
5894 .assets
5895 .get_asset_by_name(requested_asset.study_id, requested_asset.name.as_str())
5896 .await
5897 .map_err(|error| core_error("lookup asset by name", error))?
5898 {
5899 return Err(AppError::Validation(format!(
5900 "asset already exists in study {}: {} ({})",
5901 requested_asset.study_id.0,
5902 requested_asset.name.as_str(),
5903 existing.asset_id.0
5904 )));
5905 }
5906
5907 let asset = self
5908 .assets
5909 .create_asset(&requested_asset)
5910 .await
5911 .map_err(|error| core_error("create asset", error))?;
5912 self.attach_tags(TagAttachmentTarget::Asset(asset.asset_id), extraction.tags)
5913 .await?;
5914
5915 self.get_asset(GetAssetRequest {
5916 asset_id: Some(asset.asset_id),
5917 study_id: None,
5918 asset_name: None,
5919 })
5920 .await
5921 }
5922
5923 pub async fn create_asset_version(
5924 &self,
5925 request: CreateAssetVersionRequest,
5926 ) -> Result<AssetVersionCatalog, AppError> {
5927 let asset = self
5928 .assets
5929 .get_asset_by_id(request.version.asset_id)
5930 .await
5931 .map_err(|error| core_error("lookup asset for version", error))?
5932 .ok_or_else(|| {
5933 AppError::Validation(format!(
5934 "asset not found for version: {}",
5935 request.version.asset_id.0
5936 ))
5937 })?;
5938
5939 let existing_versions = self
5940 .assets
5941 .list_asset_versions(asset.asset_id)
5942 .await
5943 .map_err(|error| core_error("list existing asset versions", error))?;
5944 ahri_tre_core::validate_new_asset_version(&existing_versions, &request.version)
5945 .map_err(|error| core_error("validate asset version", error))?;
5946
5947 let extraction =
5948 ahri_tre_core::extract_inline_tags(request.version.version_note.as_deref());
5949 let mut requested_version = request.version;
5950 requested_version.version_note = extraction.cleaned_text;
5951 let version = ahri_tre_core::prepare_asset_version_for_create(requested_version)
5952 .map_err(|error| core_error("prepare asset version", error))?;
5953 let saved = self
5954 .assets
5955 .admit_asset_version(&version, &request.classification)
5956 .await
5957 .map_err(|error| core_error("save asset version", error))?;
5958 self.attach_tags(
5959 TagAttachmentTarget::AssetVersion(saved.version_id),
5960 extraction.tags,
5961 )
5962 .await?;
5963
5964 let catalog = self
5965 .get_asset(GetAssetRequest {
5966 asset_id: Some(asset.asset_id),
5967 study_id: None,
5968 asset_name: None,
5969 })
5970 .await?;
5971
5972 match catalog.latest_version.as_ref() {
5973 Some(latest) if latest.version_id == saved.version_id => Ok(catalog),
5974 Some(latest) => Err(AppError::Infrastructure(format!(
5975 "asset latest version did not move to newly created version {}; current latest is {}",
5976 saved.version_id.0, latest.version_id.0
5977 ))),
5978 None => Err(AppError::Infrastructure(format!(
5979 "asset latest version was not set after creating version {}",
5980 saved.version_id.0
5981 ))),
5982 }
5983 }
5984
5985 pub async fn create_new_asset_version(
5986 &self,
5987 request: CreateNewAssetVersionRequest,
5988 ) -> Result<AssetVersionCatalog, AppError> {
5989 let asset = self
5990 .assets
5991 .get_asset_by_id(request.version.asset_id)
5992 .await
5993 .map_err(|error| core_error("lookup asset for version", error))?
5994 .ok_or_else(|| {
5995 AppError::Validation(format!(
5996 "asset not found for version: {}",
5997 request.version.asset_id.0
5998 ))
5999 })?;
6000
6001 let existing_versions = self
6002 .assets
6003 .list_asset_versions(asset.asset_id)
6004 .await
6005 .map_err(|error| core_error("list existing asset versions", error))?;
6006 ahri_tre_core::validate_new_asset_version_input(&existing_versions, &request.version)
6007 .map_err(|error| core_error("validate asset version", error))?;
6008
6009 let extraction =
6010 ahri_tre_core::extract_inline_tags(request.version.version_note.as_deref());
6011 let mut requested_version = request.version;
6012 requested_version.version_note = extraction.cleaned_text;
6013 let version = ahri_tre_core::prepare_new_asset_version_for_create(requested_version)
6014 .map_err(|error| core_error("prepare asset version", error))?;
6015 let saved = self
6016 .assets
6017 .create_asset_version(&version)
6018 .await
6019 .map_err(|error| core_error("create asset version", error))?;
6020 self.attach_tags(
6021 TagAttachmentTarget::AssetVersion(saved.version_id),
6022 extraction.tags,
6023 )
6024 .await?;
6025
6026 let catalog = self
6027 .get_asset(GetAssetRequest {
6028 asset_id: Some(asset.asset_id),
6029 study_id: None,
6030 asset_name: None,
6031 })
6032 .await?;
6033
6034 match catalog.latest_version.as_ref() {
6035 Some(latest) if latest.version_id == saved.version_id => Ok(catalog),
6036 Some(latest) => Err(AppError::Infrastructure(format!(
6037 "asset latest version did not move to newly created version {}; current latest is {}",
6038 saved.version_id.0, latest.version_id.0
6039 ))),
6040 None => Err(AppError::Infrastructure(format!(
6041 "asset latest version was not set after creating version {}",
6042 saved.version_id.0
6043 ))),
6044 }
6045 }
6046
6047 pub async fn list_study_assets(
6048 &self,
6049 request: ListStudyAssetsRequest,
6050 ) -> Result<Vec<ahri_tre_types::AssetRecord>, AppError> {
6051 self.studies
6052 .get_study_by_id(request.study_id)
6053 .await
6054 .map_err(|error| core_error("lookup study for asset list", error))?
6055 .ok_or_else(|| {
6056 AppError::Validation(format!("asset study not found: {}", request.study_id.0))
6057 })?;
6058
6059 self.assets
6060 .list_assets_for_study(request.study_id)
6061 .await
6062 .map_err(|error| core_error("list study assets", error))
6063 }
6064
6065 pub async fn list_study_datasets(
6066 &self,
6067 request: ListStudyDatasetsRequest,
6068 ) -> Result<StudyDatasetList, AppError> {
6069 let mut datasets = Vec::new();
6070 for asset in self
6071 .list_study_assets(ListStudyAssetsRequest {
6072 study_id: request.study_id,
6073 })
6074 .await?
6075 .into_iter()
6076 .filter(|asset| asset.asset_type == ahri_tre_types::AssetType::Dataset)
6077 {
6078 let mut catalog = self.complete_dataset_catalog(asset).await?;
6079 let Some(latest_version) = catalog.latest_version.as_ref() else {
6080 continue;
6081 };
6082 let latest_dataset = self
6083 .assets
6084 .get_dataset(latest_version.version_id)
6085 .await
6086 .map_err(|error| core_error("lookup latest dataset record", error))?;
6087 if latest_dataset.is_none() {
6088 continue;
6089 }
6090 if !request.include_versions {
6091 catalog.versions.clear();
6092 }
6093 datasets.push(DatasetListItem {
6094 catalog,
6095 latest_dataset,
6096 });
6097 }
6098 Ok(StudyDatasetList {
6099 study_id: request.study_id,
6100 datasets,
6101 })
6102 }
6103
6104 pub async fn get_study_dataset(
6105 &self,
6106 request: GetStudyDatasetRequest,
6107 ) -> Result<Option<StudyDatasetMetadata>, AppError> {
6108 self.studies
6109 .get_study_by_id(request.study_id)
6110 .await
6111 .map_err(|error| core_error("lookup study for dataset metadata", error))?
6112 .ok_or_else(|| {
6113 AppError::Validation(format!("dataset study not found: {}", request.study_id.0))
6114 })?;
6115
6116 let Some(asset) = self
6117 .assets
6118 .get_asset_by_name(request.study_id, request.dataset_name.as_str())
6119 .await
6120 .map_err(|error| core_error("lookup dataset asset by name", error))?
6121 else {
6122 return Ok(None);
6123 };
6124 if asset.asset_type != ahri_tre_types::AssetType::Dataset {
6125 return Ok(None);
6126 }
6127 let catalog = self.complete_dataset_catalog(asset).await?;
6128 let latest_version = catalog.latest_version.as_ref().ok_or_else(|| {
6129 AppError::Validation(format!(
6130 "dataset {} has no complete versions",
6131 request.dataset_name.as_str()
6132 ))
6133 })?;
6134 let dataset = self
6135 .assets
6136 .get_dataset(latest_version.version_id)
6137 .await
6138 .map_err(|error| core_error("lookup dataset metadata record", error))?
6139 .ok_or_else(|| {
6140 AppError::Validation(format!(
6141 "dataset metadata not found: {}",
6142 latest_version.version_id.0
6143 ))
6144 })?;
6145 let variable_links = self
6146 .variables
6147 .list_dataset_variables(dataset.dataset_id)
6148 .await
6149 .map_err(|error| core_error("list dataset variable links", error))?;
6150 let variable_count = variable_links.len();
6151 let variables = if request.with_variables {
6152 let mut variables = Vec::with_capacity(variable_links.len());
6153 for link in variable_links {
6154 variables.push(self.dataset_variable_metadata(link).await?);
6155 }
6156 variables
6157 } else {
6158 Vec::new()
6159 };
6160 Ok(Some(StudyDatasetMetadata {
6161 catalog,
6162 dataset,
6163 variables_included: request.with_variables,
6164 variable_count,
6165 variables,
6166 }))
6167 }
6168
6169 pub async fn get_dataset_metadata_by_id(
6170 &self,
6171 request: DatasetMetadataByIdRequest,
6172 ) -> Result<StudyDatasetMetadata, AppError> {
6173 let asset = self
6174 .assets
6175 .get_asset_by_id(request.dataset_id)
6176 .await
6177 .map_err(|error| core_error("lookup dataset asset by id", error))?
6178 .ok_or_else(|| {
6179 AppError::Validation(format!("dataset not found: {}", request.dataset_id.0))
6180 })?;
6181 if asset.asset_type != ahri_tre_types::AssetType::Dataset {
6182 return Err(AppError::Validation(format!(
6183 "asset {} is not a dataset",
6184 asset.name.as_str()
6185 )));
6186 }
6187 let catalog = self.complete_dataset_catalog(asset).await?;
6188 let latest_version = catalog.latest_version.as_ref().ok_or_else(|| {
6189 AppError::Validation(format!(
6190 "dataset {} has no complete versions",
6191 catalog.asset.name.as_str()
6192 ))
6193 })?;
6194 let dataset = self
6195 .assets
6196 .get_dataset(latest_version.version_id)
6197 .await
6198 .map_err(|error| core_error("lookup dataset metadata record", error))?
6199 .ok_or_else(|| {
6200 AppError::Validation(format!(
6201 "dataset metadata not found: {}",
6202 latest_version.version_id.0
6203 ))
6204 })?;
6205 let variable_links = self
6206 .variables
6207 .list_dataset_variables(dataset.dataset_id)
6208 .await
6209 .map_err(|error| core_error("list dataset variable links", error))?;
6210 let variable_count = variable_links.len();
6211 let variables = if request.with_variables {
6212 let mut variables = Vec::with_capacity(variable_links.len());
6213 for link in variable_links {
6214 variables.push(self.dataset_variable_metadata(link).await?);
6215 }
6216 variables
6217 } else {
6218 Vec::new()
6219 };
6220 Ok(StudyDatasetMetadata {
6221 catalog,
6222 dataset,
6223 variables_included: request.with_variables,
6224 variable_count,
6225 variables,
6226 })
6227 }
6228
6229 pub async fn read_dataset_metadata(
6230 &self,
6231 request: ReadDatasetMetadataRequest,
6232 ) -> Result<Option<DatasetMetadataResult>, AppError> {
6233 let with_variables = request.with_variables;
6234 let (study, metadata) = match request.dataset {
6235 DatasetMetadataSelector::Name { study, name } => {
6236 let Some(study) = self.get_study(GetStudyRequest { study }).await? else {
6237 return Ok(None);
6238 };
6239 let Some(metadata) = self
6240 .get_study_dataset(GetStudyDatasetRequest {
6241 study_id: study.study.study_id,
6242 dataset_name: name,
6243 with_variables,
6244 })
6245 .await?
6246 else {
6247 return Ok(None);
6248 };
6249 (study.study, metadata)
6250 }
6251 DatasetMetadataSelector::Id { dataset_id } => {
6252 let metadata = self
6253 .get_dataset_metadata_by_id(DatasetMetadataByIdRequest {
6254 dataset_id,
6255 with_variables,
6256 })
6257 .await?;
6258 let study = self
6259 .require_study_id(metadata.catalog.asset.study_id)
6260 .await?;
6261 (study, metadata)
6262 }
6263 };
6264 Ok(Some(DatasetMetadataResult { study, metadata }))
6265 }
6266
6267 pub async fn preview_study_dataset(
6268 &self,
6269 session: &DataStoreSession,
6270 request: PreviewStudyDatasetRequest,
6271 ) -> Result<Option<DatasetPreview>, AppError> {
6272 let Some(metadata) = self
6273 .get_study_dataset(GetStudyDatasetRequest {
6274 study_id: request.study_id,
6275 dataset_name: request.dataset_name,
6276 with_variables: false,
6277 })
6278 .await?
6279 else {
6280 return Ok(None);
6281 };
6282 let latest_version = metadata.catalog.latest_version.as_ref().ok_or_else(|| {
6283 AppError::Validation(format!(
6284 "dataset {} has no complete versions",
6285 metadata.catalog.asset.name.as_str()
6286 ))
6287 })?;
6288
6289 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
6290 let preview = lake
6291 .preview_dataset_table(
6292 session.lake_connection(),
6293 &PreviewDatasetTableRequest {
6294 study_id: metadata.catalog.asset.study_id,
6295 dataset_name: metadata.catalog.asset.name.clone(),
6296 major: latest_version.major,
6297 minor: latest_version.minor,
6298 patch: latest_version.patch,
6299 limit: request.limit,
6300 },
6301 )
6302 .map_err(|error| infrastructure_error("preview dataset from DuckLake", error))?;
6303
6304 Ok(Some(DatasetPreview {
6305 catalog: metadata.catalog,
6306 dataset: metadata.dataset,
6307 columns: preview
6308 .columns
6309 .into_iter()
6310 .map(|column| DatasetPreviewColumn {
6311 name: column.name,
6312 value_type: column.duckdb_type,
6313 })
6314 .collect(),
6315 rows: preview.rows,
6316 displayed_row_count: preview.displayed_row_count,
6317 total_row_count: preview.total_row_count,
6318 truncated: preview.truncated,
6319 limit: request.limit,
6320 }))
6321 }
6322
6323 pub async fn preview_dataset_by_id(
6324 &self,
6325 session: &DataStoreSession,
6326 request: PreviewDatasetByIdRequest,
6327 ) -> Result<DatasetPreview, AppError> {
6328 let metadata = self
6329 .get_dataset_metadata_by_id(DatasetMetadataByIdRequest {
6330 dataset_id: request.dataset_id,
6331 with_variables: false,
6332 })
6333 .await?;
6334 let latest_version = metadata.catalog.latest_version.as_ref().ok_or_else(|| {
6335 AppError::Validation(format!(
6336 "dataset {} has no complete versions",
6337 metadata.catalog.asset.name.as_str()
6338 ))
6339 })?;
6340
6341 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
6342 let preview = lake
6343 .preview_dataset_table(
6344 session.lake_connection(),
6345 &PreviewDatasetTableRequest {
6346 study_id: metadata.catalog.asset.study_id,
6347 dataset_name: metadata.catalog.asset.name.clone(),
6348 major: latest_version.major,
6349 minor: latest_version.minor,
6350 patch: latest_version.patch,
6351 limit: request.limit,
6352 },
6353 )
6354 .map_err(|error| infrastructure_error("preview dataset from DuckLake", error))?;
6355
6356 Ok(DatasetPreview {
6357 catalog: metadata.catalog,
6358 dataset: metadata.dataset,
6359 columns: preview
6360 .columns
6361 .into_iter()
6362 .map(|column| DatasetPreviewColumn {
6363 name: column.name,
6364 value_type: column.duckdb_type,
6365 })
6366 .collect(),
6367 rows: preview.rows,
6368 displayed_row_count: preview.displayed_row_count,
6369 total_row_count: preview.total_row_count,
6370 truncated: preview.truncated,
6371 limit: request.limit,
6372 })
6373 }
6374
6375 pub async fn withdraw_dataset_version(
6376 &self,
6377 session: Option<&mut DataStoreSession>,
6378 request: WithdrawDatasetVersionRequest,
6379 ) -> Result<WithdrawDatasetVersionResult, AppError> {
6380 self.withdraw_resolved_dataset_version(session, request, None)
6381 .await
6382 }
6383
6384 async fn withdraw_resolved_dataset_version(
6385 &self,
6386 session: Option<&mut DataStoreSession>,
6387 request: WithdrawDatasetVersionRequest,
6388 resolved_version: Option<ahri_tre_types::VersionId>,
6389 ) -> Result<WithdrawDatasetVersionResult, AppError> {
6390 let reason = request.reason.trim().to_string();
6391 if reason.is_empty() {
6392 return Err(AppError::Validation(
6393 "dataset withdrawal reason must not be empty".to_string(),
6394 ));
6395 }
6396 if !request.force {
6397 return Err(AppError::Validation(
6398 "withdrawing a completed dataset version requires --force".to_string(),
6399 ));
6400 }
6401 if request.version.trim().eq_ignore_ascii_case("latest") {
6402 return Err(AppError::Validation(
6403 "dataset withdrawal requires an explicit version x.y.z, not latest".to_string(),
6404 ));
6405 }
6406 if parse_semver_selector(request.version.trim()).is_none() {
6407 return Err(AppError::Validation(format!(
6408 "dataset withdrawal version must be semver x.y.z: {}",
6409 request.version.trim()
6410 )));
6411 }
6412
6413 self.studies
6414 .get_study_by_id(request.study_id)
6415 .await
6416 .map_err(|error| core_error("lookup study for dataset withdrawal", error))?
6417 .ok_or_else(|| {
6418 AppError::Validation(format!("dataset study not found: {}", request.study_id.0))
6419 })?;
6420 self.authorize_study_asset_lifecycle_for_app(
6421 request.study_id,
6422 request.actor.as_deref(),
6423 "dataset withdrawal",
6424 )
6425 .await?;
6426 let asset = self
6427 .assets
6428 .get_asset_by_name(request.study_id, request.dataset_name.as_str())
6429 .await
6430 .map_err(|error| core_error("lookup dataset asset for withdrawal", error))?
6431 .ok_or_else(|| {
6432 AppError::Validation(format!(
6433 "dataset not found: {}",
6434 request.dataset_name.as_str()
6435 ))
6436 })?;
6437 if asset.asset_type != ahri_tre_types::AssetType::Dataset {
6438 return Err(AppError::Validation(format!(
6439 "asset {} is not a dataset",
6440 asset.name.as_str()
6441 )));
6442 }
6443
6444 let versions = self
6445 .assets
6446 .list_asset_versions(asset.asset_id)
6447 .await
6448 .map_err(|error| core_error("list dataset versions for withdrawal", error))?;
6449 let latest_version = versions
6450 .iter()
6451 .find(|version| version.is_latest == Some(true))
6452 .cloned();
6453 let full_catalog = AssetVersionCatalog {
6454 asset: asset.clone(),
6455 versions,
6456 latest_version,
6457 };
6458 let withdrawn_version =
6459 resolve_requested_asset_version(&full_catalog, Some(request.version.as_str()))?;
6460 if resolved_version.is_some_and(|id| id != withdrawn_version.version_id) {
6461 return Err(AppError::Conflict(
6462 "Withdrawal target changed after resolution".into(),
6463 ));
6464 }
6465 let dataset = self
6466 .assets
6467 .get_dataset(withdrawn_version.version_id)
6468 .await
6469 .map_err(|error| core_error("lookup dataset record for withdrawal", error))?
6470 .ok_or_else(|| {
6471 AppError::Validation(format!(
6472 "dataset version is not complete: {} {}",
6473 asset.name.as_str(),
6474 request.version.trim()
6475 ))
6476 })?;
6477
6478 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
6479 request.study_id,
6480 asset.name.as_str(),
6481 withdrawn_version.major,
6482 withdrawn_version.minor,
6483 withdrawn_version.patch,
6484 );
6485 let existing = self
6486 .assets
6487 .get_dataset_version_withdrawal(dataset.dataset_id)
6488 .await
6489 .map_err(|error| core_error("lookup dataset version withdrawal", error))?;
6490 let already_withdrawn = existing.is_some();
6491 let withdrawal = match existing {
6492 Some(withdrawal) => withdrawal,
6493 None => {
6494 let withdrawal = ahri_tre_types::NewDatasetVersionWithdrawalRecord {
6495 dataset_id: dataset.dataset_id,
6496 asset_id: asset.asset_id,
6497 withdrawn_by: request.actor.clone(),
6498 reason,
6499 tombstone_state: "withdrawn".to_string(),
6500 provenance: Some(format!(
6501 "withdrawn dataset={} version={} lake_relation={}",
6502 asset.name.as_str(),
6503 asset_version_label_text(&withdrawn_version),
6504 relation.qualified_name()
6505 )),
6506 drop_lake_table: request.drop_lake_table,
6507 };
6508 self.assets
6509 .save_dataset_version_withdrawal(&withdrawal)
6510 .await
6511 .map_err(|error| core_error("save dataset version withdrawal", error))?
6512 }
6513 };
6514
6515 if !already_withdrawn {
6516 self.assets
6517 .deactivate_dataset_version_links(dataset.dataset_id)
6518 .await
6519 .map_err(|error| core_error("deactivate dataset version links", error))?;
6520 }
6521
6522 let promoted_latest_version = self
6523 .repair_latest_after_withdrawal(
6524 asset.asset_id,
6525 withdrawn_version.version_id,
6526 withdrawn_version.is_latest == Some(true),
6527 )
6528 .await?;
6529
6530 let lake_table_drop = if already_withdrawn {
6531 LakeTableDropDisposition::AlreadyWithdrawn
6532 } else if request.drop_lake_table {
6533 let session = session.ok_or_else(|| {
6534 AppError::Validation(
6535 "dropping the lake table requires an open datastore session".to_string(),
6536 )
6537 })?;
6538 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
6539 lake.drop_dataset_table(session.lake_connection(), &relation)
6540 .map_err(|error| infrastructure_error("drop withdrawn dataset table", error))?;
6541 LakeTableDropDisposition::DroppedIfExisted
6542 } else {
6543 LakeTableDropDisposition::NotRequested
6544 };
6545
6546 let catalog = self.complete_dataset_catalog(asset).await?;
6547 Ok(WithdrawDatasetVersionResult {
6548 catalog,
6549 withdrawn_version,
6550 withdrawal,
6551 promoted_latest_version,
6552 lake_relation: relation.qualified_name(),
6553 lake_schema: relation.schema_name,
6554 lake_table: relation.table_name,
6555 lake_table_drop,
6556 already_withdrawn,
6557 })
6558 }
6559
6560 pub async fn plan_archive_delete(
6561 &self,
6562 request: PlanArchiveDeleteRequest,
6563 ) -> Result<ArchiveDeletePlanningResult, AppError> {
6564 let reason = request.reason.trim().to_string();
6565 let actor = request
6566 .actor
6567 .as_deref()
6568 .map(str::trim)
6569 .filter(|actor| !actor.is_empty())
6570 .map(str::to_string);
6571 let mut validation_errors = Vec::new();
6572 if reason.is_empty() {
6573 validation_errors.push(archive_delete_validation_error(
6574 "missing_reason",
6575 "archive/delete reason must not be empty",
6576 ));
6577 }
6578 if actor.is_none() {
6579 validation_errors.push(archive_delete_validation_error(
6580 "missing_actor",
6581 "archive/delete planning requires an authenticated TRE user or explicit actor",
6582 ));
6583 }
6584
6585 let resolved_version = match request.version_target {
6586 Some(ArchiveDeleteVersionTarget::Version { version_id }) => Some(version_id),
6587 Some(ArchiveDeleteVersionTarget::Latest) => {
6588 validation_errors.push(archive_delete_validation_error(
6589 "unresolved_latest_version",
6590 "archive/delete requires a concrete asset version, not latest",
6591 ));
6592 None
6593 }
6594 None => None,
6595 };
6596
6597 if matches!(request.target, DeleteTarget::AssetVersion(_)) && request.delete_all_versions {
6598 validation_errors.push(archive_delete_validation_error(
6599 "version_target_all_versions_conflict",
6600 "asset-version archive/delete cannot also request all versions",
6601 ));
6602 }
6603 if let DeleteTarget::AssetVersion(version_id) = request.target
6604 && let Some(resolved_version) = resolved_version
6605 && resolved_version != version_id
6606 {
6607 validation_errors.push(archive_delete_validation_error(
6608 "version_target_mismatch",
6609 "asset-version target must match the concrete version selector",
6610 ));
6611 }
6612
6613 if !validation_errors.is_empty() {
6614 return Ok(ArchiveDeletePlanningResult::rejected(validation_errors));
6615 }
6616
6617 let authorization = match self
6618 .authorize_archive_delete_plan(
6619 &request.target,
6620 actor.as_deref().expect("actor checked"),
6621 )
6622 .await
6623 {
6624 Ok(authorization) => authorization,
6625 Err(error) => return Ok(ArchiveDeletePlanningResult::rejected(vec![error])),
6626 };
6627
6628 let policy = DeletePolicy {
6629 archive_mode: request.archive_mode.clone(),
6630 cascade: request.cascade,
6631 force: request.force,
6632 };
6633 let delete_request = DeleteRequest {
6634 target: request.target.clone(),
6635 version: resolved_version,
6636 delete_all_versions: request.delete_all_versions,
6637 };
6638 let core_policy = DefaultArchiveDeletePolicy;
6639 let core_plan = match core_policy.plan_delete(&delete_request, &policy) {
6640 Ok(plan) => plan,
6641 Err(CoreError::Validation(message)) => {
6642 return Ok(ArchiveDeletePlanningResult::rejected(vec![
6643 archive_delete_validation_error("policy_validation_failed", message),
6644 ]));
6645 }
6646 Err(error) => return Err(core_error("plan archive/delete policy", error)),
6647 };
6648
6649 Ok(ArchiveDeletePlanningResult::accepted(ArchiveDeletePlan {
6650 target: core_plan.target,
6651 archive_mode: request.archive_mode,
6652 archive_required: core_plan.archive_before_delete,
6653 authorization,
6654 cascade: core_plan.cascade,
6655 force: request.force,
6656 reason,
6657 actor,
6658 resolved_version,
6659 delete_all_versions: request.delete_all_versions,
6660 dependency_summary: ArchiveDeleteDependencySummary::not_analyzed(),
6661 archive_preview: archive_delete_archive_preview(core_plan.archive_before_delete),
6662 metadata_cleanup_preview: ArchiveDeleteStagePreview::pending(
6663 true,
6664 "metadata cleanup will be resolved by the executable archive/delete workflow",
6665 ),
6666 lake_cleanup_preview: ArchiveDeleteStagePreview::pending(
6667 true,
6668 "lake cleanup will be resolved by the executable archive/delete workflow",
6669 ),
6670 retry_state: ArchiveDeleteRetryState::NotStarted,
6671 }))
6672 }
6673
6674 pub async fn delete_dataset_asset_version(
6675 &self,
6676 session: &mut DataStoreSession,
6677 request: DeleteDatasetAssetVersionRequest,
6678 ) -> Result<DeleteDatasetAssetVersionResult, AppError> {
6679 let DeleteTarget::AssetVersion(version_id) = request.plan.target else {
6680 return Err(AppError::Validation(
6681 "dataset asset-version delete requires an asset-version plan".to_string(),
6682 ));
6683 };
6684 if request.plan.resolved_version != Some(version_id) {
6685 return Err(AppError::Validation(
6686 "dataset asset-version delete requires a concrete resolved version".to_string(),
6687 ));
6688 }
6689 if request.plan.delete_all_versions {
6690 return Err(AppError::Validation(
6691 "dataset asset-version delete cannot request all versions".to_string(),
6692 ));
6693 }
6694 if !matches!(
6695 request.plan.archive_mode,
6696 ArchiveMode::AssetArchive | ArchiveMode::None
6697 ) {
6698 return Err(AppError::Validation(
6699 "dataset asset-version delete requires asset archive or explicit no-archive mode"
6700 .to_string(),
6701 ));
6702 }
6703
6704 let catalog = self
6705 .catalog_for_version(version_id)
6706 .await
6707 .map_err(|error| core_error("lookup dataset asset-version delete target", error))?;
6708 if catalog.asset.asset_type != ahri_tre_types::AssetType::Dataset {
6709 return Err(AppError::Validation(format!(
6710 "asset version {} belongs to non-dataset asset {}",
6711 version_id.0,
6712 catalog.asset.name.as_str()
6713 )));
6714 }
6715 let target_version = catalog
6716 .versions
6717 .iter()
6718 .find(|version| version.version_id == version_id)
6719 .cloned()
6720 .ok_or_else(|| {
6721 AppError::Validation(format!("asset version not found: {}", version_id.0))
6722 })?;
6723 self.assets
6724 .get_dataset(version_id)
6725 .await
6726 .map_err(|error| core_error("lookup dataset deletion tombstone target", error))?
6727 .ok_or_else(|| {
6728 AppError::Validation(format!(
6729 "dataset version is not complete: {} {}",
6730 catalog.asset.name.as_str(),
6731 asset_version_label_text(&target_version)
6732 ))
6733 })?;
6734
6735 let deletion = self
6736 .withdraw_resolved_dataset_version(
6737 Some(session),
6738 WithdrawDatasetVersionRequest {
6739 study_id: catalog.asset.study_id,
6740 dataset_name: catalog.asset.name.clone(),
6741 version: format!(
6742 "{}.{}.{}",
6743 target_version.major, target_version.minor, target_version.patch
6744 ),
6745 reason: request.plan.reason.clone(),
6746 actor: request.plan.actor.clone(),
6747 force: true,
6748 drop_lake_table: true,
6749 },
6750 Some(version_id),
6751 )
6752 .await?;
6753 Ok(DeleteDatasetAssetVersionResult {
6754 plan: request.plan,
6755 target_version,
6756 tombstone: deletion.withdrawal.clone(),
6757 promoted_latest_version: deletion.promoted_latest_version.clone(),
6758 lake_cleanup: deletion.lake_table_drop,
6759 retry_state: ArchiveDeleteRetryState::Completed,
6760 deletion,
6761 })
6762 }
6763
6764 pub async fn delete_datafile_asset_version(
6765 &self,
6766 session: &mut DataStoreSession,
6767 request: DeleteDataFileAssetVersionRequest,
6768 ) -> Result<DeleteDataFileAssetVersionResult, AppError> {
6769 let DeleteTarget::AssetVersion(version_id) = request.plan.target else {
6770 return Err(AppError::Validation(
6771 "datafile asset-version delete requires an asset-version plan".to_string(),
6772 ));
6773 };
6774 if request.plan.resolved_version != Some(version_id) {
6775 return Err(AppError::Validation(
6776 "datafile asset-version delete requires a concrete resolved version".to_string(),
6777 ));
6778 }
6779 if request.plan.delete_all_versions {
6780 return Err(AppError::Validation(
6781 "datafile asset-version delete cannot request all versions".to_string(),
6782 ));
6783 }
6784
6785 let catalog = self
6786 .catalog_for_version(version_id)
6787 .await
6788 .map_err(|error| core_error("lookup datafile asset-version delete target", error))?;
6789 if catalog.asset.asset_type != ahri_tre_types::AssetType::File {
6790 return Err(AppError::Validation(format!(
6791 "asset version {} belongs to non-datafile asset {}",
6792 version_id.0,
6793 catalog.asset.name.as_str()
6794 )));
6795 }
6796 let target_version = catalog
6797 .versions
6798 .iter()
6799 .find(|version| version.version_id == version_id)
6800 .cloned()
6801 .ok_or_else(|| {
6802 AppError::Validation(format!("asset version not found: {}", version_id.0))
6803 })?;
6804 let datafile = self
6805 .assets
6806 .get_datafile(version_id)
6807 .await
6808 .map_err(|error| core_error("lookup datafile deletion target", error))?
6809 .ok_or_else(|| {
6810 AppError::Validation(format!(
6811 "datafile version is not complete: {} {}",
6812 catalog.asset.name.as_str(),
6813 asset_version_label_text(&target_version)
6814 ))
6815 })?;
6816
6817 let promoted_latest_version = self
6818 .datafile_delete_latest_candidate(
6819 catalog.asset.asset_id,
6820 target_version.version_id,
6821 target_version.is_latest == Some(true),
6822 )
6823 .await?;
6824 let removed = DuckLakeAdapter::new(&session.runtime().lake.data_path)
6825 .delete_datafile(&DeleteDataFileRequest {
6826 datafile: datafile.clone(),
6827 })
6828 .map_err(|error| infrastructure_error("delete managed datafile payload", error))?;
6829 self.assets
6830 .delete_datafile_version_metadata(
6831 version_id,
6832 (target_version.is_latest == Some(true)).then_some((
6833 catalog.asset.asset_id,
6834 promoted_latest_version
6835 .as_ref()
6836 .map(|version| version.version_id),
6837 )),
6838 )
6839 .await
6840 .map_err(|error| core_error("delete active Datafile version metadata", error))?;
6841
6842 Ok(DeleteDataFileAssetVersionResult {
6843 plan: request.plan,
6844 asset: catalog.asset,
6845 target_version,
6846 datafile,
6847 promoted_latest_version,
6848 managed_file_cleanup: if removed.existed {
6849 ArchiveDeleteCleanupDisposition::Completed
6850 } else {
6851 ArchiveDeleteCleanupDisposition::Skipped
6852 },
6853 metadata_cleanup: ArchiveDeleteCleanupDisposition::Completed,
6854 removed_size_bytes: removed.removed_size_bytes,
6855 retry_state: ArchiveDeleteRetryState::Completed,
6856 })
6857 }
6858
6859 pub async fn delete_asset(
6860 &self,
6861 session: &mut DataStoreSession,
6862 request: DeleteAssetRequest,
6863 ) -> Result<DeleteAssetResult, AppError> {
6864 let DeleteTarget::Asset(asset_id) = request.plan.target else {
6865 return Err(AppError::Validation(
6866 "whole-asset delete requires an asset plan".to_string(),
6867 ));
6868 };
6869 if !request.plan.cascade || !request.plan.delete_all_versions {
6870 return Err(AppError::Validation(
6871 "whole-asset delete requires cascade and all-version policy".to_string(),
6872 ));
6873 }
6874 let asset = self
6875 .assets
6876 .get_asset_by_id(asset_id)
6877 .await
6878 .map_err(|error| core_error("lookup asset delete target", error))?
6879 .ok_or_else(|| AppError::Validation(format!("asset not found: {}", asset_id.0)))?;
6880 let versions = self
6881 .assets
6882 .list_asset_versions(asset_id)
6883 .await
6884 .map_err(|error| core_error("list asset versions for delete", error))?;
6885 let mut version_results = Vec::new();
6886 let mut failed_versions = 0;
6887 for version in versions {
6888 let subplan = ArchiveDeletePlan {
6889 target: DeleteTarget::AssetVersion(version.version_id),
6890 resolved_version: Some(version.version_id),
6891 delete_all_versions: false,
6892 cascade: request.plan.cascade,
6893 ..request.plan.clone()
6894 };
6895 let result = match asset.asset_type {
6896 ahri_tre_types::AssetType::Dataset => self
6897 .delete_dataset_asset_version(
6898 session,
6899 DeleteDatasetAssetVersionRequest { plan: subplan },
6900 )
6901 .await
6902 .map(Box::new)
6903 .map(DeleteAssetVersionResult::Dataset),
6904 ahri_tre_types::AssetType::File => self
6905 .delete_datafile_asset_version(
6906 session,
6907 DeleteDataFileAssetVersionRequest { plan: subplan },
6908 )
6909 .await
6910 .map(Box::new)
6911 .map(DeleteAssetVersionResult::Datafile),
6912 };
6913 match result {
6914 Ok(result) => version_results.push(result),
6915 Err(error) => {
6916 failed_versions += 1;
6917 version_results.push(DeleteAssetVersionResult::Skipped {
6918 version,
6919 reason: error.to_string(),
6920 });
6921 }
6922 }
6923 }
6924 if failed_versions == 0 {
6925 self.assets
6926 .delete_asset_record(asset_id)
6927 .await
6928 .map_err(|error| core_error("delete active asset metadata", error))?;
6929 }
6930 let deleted_versions = version_results
6931 .iter()
6932 .filter(|result| !matches!(result, DeleteAssetVersionResult::Skipped { .. }))
6933 .count();
6934 Ok(DeleteAssetResult {
6935 plan: request.plan,
6936 asset,
6937 skipped_versions: version_results.len() - deleted_versions,
6938 deleted_versions,
6939 failed_versions,
6940 version_results,
6941 metadata_cleanup: if failed_versions == 0 {
6942 ArchiveDeleteCleanupDisposition::Completed
6943 } else {
6944 ArchiveDeleteCleanupDisposition::FailedRetryable
6945 },
6946 retry_state: if failed_versions == 0 {
6947 ArchiveDeleteRetryState::Completed
6948 } else {
6949 ArchiveDeleteRetryState::Retryable
6950 },
6951 })
6952 }
6953
6954 pub async fn delete_archived_study(
6955 &self,
6956 session: &mut DataStoreSession,
6957 request: DeleteArchivedStudyRequest,
6958 ) -> Result<DeleteArchivedStudyResult, AppError> {
6959 let DeleteTarget::Study(study_id) = request.plan.target else {
6960 return Err(AppError::Validation(
6961 "study delete requires a study plan".to_string(),
6962 ));
6963 };
6964 if !request.plan.cascade {
6965 return Err(AppError::Validation(
6966 "study delete requires cascade after archive ingest".to_string(),
6967 ));
6968 }
6969 if request.archive.archive_datafile.datafile_id
6970 != request.archive.archive_version.version_id
6971 {
6972 return Err(AppError::Validation(
6973 "study delete requires completed Archive-study datafile ingest".to_string(),
6974 ));
6975 }
6976 if request.archive.prepared.source_study.study_id != study_id {
6977 return Err(AppError::Validation(
6978 "study delete plan target must match archived source study".to_string(),
6979 ));
6980 }
6981
6982 let source_study = request.archive.prepared.source_study.clone();
6983 let assets = self
6984 .list_study_assets(ListStudyAssetsRequest { study_id })
6985 .await?;
6986 let mut asset_results = Vec::new();
6987 let mut failed_versions = 0;
6988 let mut skipped_versions = 0;
6989 let mut deleted_versions = 0;
6990 for asset in assets {
6991 let asset_plan = ArchiveDeletePlan {
6992 target: DeleteTarget::Asset(asset.asset_id),
6993 resolved_version: None,
6994 delete_all_versions: true,
6995 cascade: true,
6996 ..request.plan.clone()
6997 };
6998 let result = self
6999 .delete_asset(session, DeleteAssetRequest { plan: asset_plan })
7000 .await?;
7001 failed_versions += result.failed_versions;
7002 skipped_versions += result.skipped_versions;
7003 deleted_versions += result.deleted_versions;
7004 asset_results.push(result);
7005 }
7006 if failed_versions == 0 {
7007 self.studies
7008 .delete_study_record(study_id)
7009 .await
7010 .map_err(|error| core_error("delete active source study metadata", error))?;
7011 }
7012 Ok(DeleteArchivedStudyResult {
7013 plan: request.plan,
7014 archive: request.archive,
7015 source_study,
7016 deleted_assets: asset_results.len(),
7017 deleted_versions,
7018 skipped_versions,
7019 failed_versions,
7020 asset_results,
7021 metadata_cleanup: if failed_versions == 0 {
7022 ArchiveDeleteCleanupDisposition::Completed
7023 } else {
7024 ArchiveDeleteCleanupDisposition::FailedRetryable
7025 },
7026 retry_state: if failed_versions == 0 {
7027 ArchiveDeleteRetryState::Completed
7028 } else {
7029 ArchiveDeleteRetryState::Retryable
7030 },
7031 })
7032 }
7033
7034 pub async fn plan_or_prepare_study_archive_for_deletion(
7035 &self,
7036 request: PrepareStudyArchiveForDeletionRequest,
7037 ) -> Result<PreparedStudyArchiveForDeletionResult, AppError> {
7038 let reason = request.reason.trim().to_string();
7039 if reason.is_empty() {
7040 return Err(AppError::Validation(
7041 "study archive reason must not be empty".to_string(),
7042 ));
7043 }
7044 let actor = request
7045 .actor
7046 .as_deref()
7047 .map(str::trim)
7048 .filter(|actor| !actor.is_empty())
7049 .map(str::to_string)
7050 .ok_or_else(|| {
7051 AppError::Validation(
7052 "study archive requires an authenticated TRE user or explicit actor"
7053 .to_string(),
7054 )
7055 })?;
7056
7057 let source_study = self
7058 .studies
7059 .get_study_by_id(request.study_id)
7060 .await
7061 .map_err(|error| core_error("lookup source study for archive", error))?
7062 .ok_or_else(|| {
7063 AppError::Validation(format!(
7064 "study not found for archive: {}",
7065 request.study_id.0
7066 ))
7067 })?;
7068
7069 let archive_name = parse_domain_name("Archive".to_string())?;
7070 let archive_domain = self
7071 .register_new_domain(RegisterNewDomainRequest {
7072 domain: ahri_tre_types::NewDomainRecord {
7073 name: archive_name.clone(),
7074 description: Some(
7075 "System archive domain for deleted-study snapshots".to_string(),
7076 ),
7077 uri: None,
7078 },
7079 })
7080 .await?;
7081
7082 let archive_study = match self
7083 .studies
7084 .get_study_by_name(archive_name.as_str())
7085 .await
7086 .map_err(|error| core_error("lookup Archive study", error))?
7087 {
7088 Some(study) => {
7089 self.study_domains
7090 .link_study_domain(study.study_id, archive_domain.domain_id)
7091 .await
7092 .map_err(|error| core_error("link Archive study domain", error))?;
7093 study
7094 }
7095 None => {
7096 self.register_new_study(RegisterNewStudyRequest {
7097 study: ahri_tre_types::NewStudyRecord {
7098 name: archive_name,
7099 description: Some(
7100 "System archive study for deleted-study snapshots".to_string(),
7101 ),
7102 documentation: None,
7103 agent_instructions: None,
7104 external_id: Some("Archive".to_string()),
7105 study_type_id: None,
7106 date_created: None,
7107 created_by: Some(actor.clone()),
7108 },
7109 domain_ids: vec![archive_domain.domain_id],
7110 })
7111 .await?
7112 .study
7113 }
7114 };
7115
7116 let source_study_id = source_study.study_id.0.simple().to_string();
7117 let snapshot_id = uuid::Uuid::new_v4().simple().to_string();
7121 let archive_asset_name =
7122 ahri_tre_types::NcName::parse(format!("archive_study_{source_study_id}_{snapshot_id}"))
7123 .map_err(|error| {
7124 AppError::Validation(format!("archive asset name is invalid: {error}"))
7125 })?;
7126
7127 Ok(PreparedStudyArchiveForDeletionResult {
7128 source_study: source_study.clone(),
7129 archive_domain,
7130 archive_study: archive_study.clone(),
7131 archive_datafile: PreparedArchiveDatafileIdentity {
7132 archive_study_id: archive_study.study_id,
7133 archive_asset_name,
7134 archive_asset_id: None,
7135 archive_version_id: None,
7136 archive_datafile_id: None,
7137 bundle_file_name: format!("study-{}.tar.zst", source_study.study_id.0),
7138 mime_type: "application/x-tar+zstd".to_string(),
7139 compression: "zstd".to_string(),
7140 dataset_payload_format: DatasetExportFormat::Parquet,
7141 },
7142 reason,
7143 actor,
7144 archive_stage: ArchiveDeleteStagePreview::pending(
7145 true,
7146 "Archive-study datafile ingest is required before source-study deletion",
7147 ),
7148 metadata_cleanup_stage: ArchiveDeleteStagePreview::pending(
7149 false,
7150 "source-study metadata cleanup has not started",
7151 ),
7152 lake_cleanup_stage: ArchiveDeleteStagePreview::pending(
7153 false,
7154 "lake cleanup has not started",
7155 ),
7156 source_study_delete_started: false,
7157 lake_payload_created: false,
7158 })
7159 }
7160
7161 pub async fn archive_study_for_deletion(
7162 &self,
7163 session: &mut DataStoreSession,
7164 request: ArchiveStudyForDeletionRequest,
7165 ) -> Result<ArchiveStudyForDeletionResult, AppError> {
7166 let prepared = self
7167 .plan_or_prepare_study_archive_for_deletion(PrepareStudyArchiveForDeletionRequest {
7168 study_id: request.study_id,
7169 reason: request.reason,
7170 actor: request.actor,
7171 })
7172 .await?;
7173 let created_at = Utc::now();
7174 let temp_dir = study_archive_temp_dir(prepared.source_study.study_id)?;
7175 fs::create_dir_all(&temp_dir).map_err(|error| {
7176 infrastructure_error("create study archive staging directory", error)
7177 })?;
7178
7179 let archive_result = async {
7180 let domains = self
7181 .study_domains
7182 .list_domains_for_study(prepared.source_study.study_id)
7183 .await
7184 .map_err(|error| core_error("list study archive domains", error))?;
7185 let mut variables = Vec::new();
7186 let mut vocabularies = Vec::new();
7187 let mut vocabulary_items = Vec::new();
7188 let mut entities = Vec::new();
7189 let mut entity_relations = Vec::new();
7190 for domain in &domains {
7191 variables.extend(
7192 self.variables
7193 .list_domain_variables(domain.domain_id)
7194 .await
7195 .map_err(|error| core_error("list study archive variables", error))?,
7196 );
7197 let domain_vocabularies = self
7198 .vocabularies
7199 .list_vocabularies(domain.domain_id)
7200 .await
7201 .map_err(|error| core_error("list study archive vocabularies", error))?;
7202 for vocabulary in &domain_vocabularies {
7203 vocabulary_items.extend(
7204 self.vocabularies
7205 .list_vocabulary_items(vocabulary.vocabulary_id)
7206 .await
7207 .map_err(|error| {
7208 core_error("list study archive vocabulary items", error)
7209 })?,
7210 );
7211 }
7212 vocabularies.extend(domain_vocabularies);
7213 entities.extend(
7214 self.entities
7215 .list_entities(domain.domain_id)
7216 .await
7217 .map_err(|error| core_error("list study archive entities", error))?,
7218 );
7219 entity_relations.extend(
7220 self.entities
7221 .list_entity_relations(domain.domain_id)
7222 .await
7223 .map_err(|error| {
7224 core_error("list study archive entity relations", error)
7225 })?,
7226 );
7227 }
7228 let assets = self
7229 .list_study_assets(ListStudyAssetsRequest {
7230 study_id: prepared.source_study.study_id,
7231 })
7232 .await?;
7233 let mut asset_versions = Vec::new();
7234 for asset in &assets {
7235 asset_versions.extend(
7236 self.assets
7237 .list_asset_versions(asset.asset_id)
7238 .await
7239 .map_err(|error| core_error("list study archive asset versions", error))?,
7240 );
7241 }
7242
7243 let mut dataset_payloads = Vec::new();
7244 let datasets = self
7245 .list_study_datasets(ListStudyDatasetsRequest {
7246 study_id: prepared.source_study.study_id,
7247 include_versions: false,
7248 })
7249 .await?;
7250 for entry in datasets.datasets {
7251 if let (Some(version), Some(_dataset)) =
7252 (entry.catalog.latest_version.clone(), entry.latest_dataset)
7253 {
7254 let bundle_path = format!(
7255 "datasets/{}_{}_{}_{}.parquet",
7256 entry.catalog.asset.name.as_str(),
7257 version.major,
7258 version.minor,
7259 version.patch
7260 );
7261 let export_path = temp_dir.join(&bundle_path);
7262 if let Some(parent) = export_path.parent() {
7263 fs::create_dir_all(parent).map_err(|error| {
7264 infrastructure_error("create archive dataset export directory", error)
7265 })?;
7266 }
7267 let exported = self
7268 .export_dataset_to_path(
7269 session,
7270 ExportDatasetToPathRequest {
7271 dataset_id: version.version_id,
7272 target_path: export_path,
7273 format: DatasetExportFormat::Parquet,
7274 limit: None,
7275 compress: false,
7276 overwrite: request.overwrite_existing_exports,
7277 transformation: archive_stage_transformation(
7278 "export dataset for study archive",
7279 &prepared.actor,
7280 ahri_tre_types::TransformationType::Export,
7281 ),
7282 },
7283 )
7284 .await?;
7285 dataset_payloads.push((
7286 exported.target_path.clone(),
7287 ArchivedDatasetFileRecord {
7288 asset_id: entry.catalog.asset.asset_id,
7289 asset_name: entry.catalog.asset.name,
7290 version_id: version.version_id,
7291 major: version.major,
7292 minor: version.minor,
7293 patch: version.patch,
7294 bundle_path,
7295 file_format: DatasetExportFormat::Parquet,
7296 row_count: exported.row_count,
7297 column_count: exported.column_count,
7298 exported_size_bytes: exported.exported_size_bytes,
7299 },
7300 ));
7301 }
7302 }
7303
7304 let mut datafile_payloads = Vec::new();
7305 let datafiles = self
7306 .list_study_datafiles(ListStudyDataFilesRequest {
7307 study_id: prepared.source_study.study_id,
7308 include_versions: true,
7309 })
7310 .await?;
7311 for entry in datafiles {
7312 for datafile in entry.datafiles {
7313 let Some(version) = entry
7314 .catalog
7315 .versions
7316 .iter()
7317 .find(|version| version.version_id == datafile.datafile_id)
7318 .cloned()
7319 else {
7320 continue;
7321 };
7322 let export_dir = temp_dir.join("datafiles");
7323 let exported = self
7324 .export_datafile(
7325 session,
7326 ExportDataFileRequest {
7327 datafile_id: datafile.datafile_id,
7328 target_dir: export_dir.clone(),
7329 compress: false,
7330 overwrite: request.overwrite_existing_exports,
7331 transformation: archive_stage_transformation(
7332 "export datafile for study archive",
7333 &prepared.actor,
7334 ahri_tre_types::TransformationType::Export,
7335 ),
7336 },
7337 )
7338 .await?;
7339 let file_name = exported
7340 .target_path
7341 .file_name()
7342 .and_then(|name| name.to_str())
7343 .ok_or_else(|| {
7344 AppError::Infrastructure(
7345 "archive datafile export path did not include a file name"
7346 .to_string(),
7347 )
7348 })?
7349 .to_string();
7350 let bundle_path = format!("datafiles/{file_name}");
7351 datafile_payloads.push((
7352 exported.target_path.clone(),
7353 ArchivedDataFileRecord {
7354 asset_id: entry.catalog.asset.asset_id,
7355 asset_name: entry.catalog.asset.name.clone(),
7356 version_id: version.version_id,
7357 major: version.major,
7358 minor: version.minor,
7359 patch: version.patch,
7360 bundle_path,
7361 edam_format: datafile.edam_format,
7362 exported_size_bytes: exported.exported_size_bytes,
7363 },
7364 ));
7365 }
7366 }
7367
7368 let transformations = self
7369 .list_study_transformations(ListStudyTransformationsRequest {
7370 study: prepared.source_study.name.as_str().to_string(),
7371 })
7372 .await
7373 .unwrap_or_default();
7374 let domain_count = domains.len();
7375 let manifest = StudyArchiveManifest {
7376 target: StudyArchiveTarget {
7377 study_id: prepared.source_study.study_id,
7378 study_name: prepared.source_study.name.clone(),
7379 },
7380 archive: StudyArchiveRegistration {
7381 archive_study_id: prepared.archive_study.study_id,
7382 archive_asset_name: prepared.archive_datafile.archive_asset_name.clone(),
7383 bundle_file_name: prepared.archive_datafile.bundle_file_name.clone(),
7384 mime_type: prepared.archive_datafile.mime_type.clone(),
7385 compression: prepared.archive_datafile.compression.clone(),
7386 },
7387 reason: prepared.reason.clone(),
7388 actor: prepared.actor.clone(),
7389 created_at,
7390 dataset_payload_format: DatasetExportFormat::Parquet,
7391 domains,
7392 variables: variables.clone(),
7393 vocabularies: vocabularies.clone(),
7394 vocabulary_items: vocabulary_items.clone(),
7395 entities: entities.clone(),
7396 entity_relations: entity_relations.clone(),
7397 assets: assets.clone(),
7398 asset_versions: asset_versions.clone(),
7399 datasets: dataset_payloads
7400 .iter()
7401 .map(|(_, record)| record.clone())
7402 .collect(),
7403 datafiles: datafile_payloads
7404 .iter()
7405 .map(|(_, record)| record.clone())
7406 .collect(),
7407 summary: StudyArchiveSummary {
7408 asset_count: assets.len(),
7409 asset_version_count: asset_versions.len(),
7410 dataset_count: dataset_payloads.len(),
7411 datafile_count: datafile_payloads.len(),
7412 domain_count,
7413 variable_count: variables.len(),
7414 vocabulary_count: vocabularies.len(),
7415 vocabulary_item_count: vocabulary_items.len(),
7416 entity_count: entities.len(),
7417 entity_relation_count: entity_relations.len(),
7418 },
7419 };
7420 let bundle_bytes = build_study_archive_bundle(
7421 &manifest,
7422 &prepared.source_study,
7423 &assets,
7424 &asset_versions,
7425 &transformations,
7426 &dataset_payloads,
7427 &datafile_payloads,
7428 )?;
7429
7430 let archive_asset = self
7431 .assets
7432 .create_asset(&ahri_tre_types::NewAssetRecord {
7433 study_id: prepared.archive_study.study_id,
7434 name: prepared.archive_datafile.archive_asset_name.clone(),
7435 description: Some(format!(
7436 "Archive bundle for deleted-study candidate {}",
7437 prepared.source_study.name.as_str()
7438 )),
7439 agent_instructions: None,
7440 asset_type: ahri_tre_types::AssetType::File,
7441 date_created: Some(created_at),
7442 created_by: Some(prepared.actor.clone()),
7443 })
7444 .await
7445 .map_err(|error| core_error("create Archive-study archive asset", error))?;
7446 let archive_version = self
7447 .assets
7448 .create_asset_version(&ahri_tre_types::NewAssetVersionRecord {
7449 classification: ahri_tre_types::IngestClassification::derived_high(),
7450 asset_id: archive_asset.asset_id,
7451 major: 1,
7452 minor: 0,
7453 patch: 0,
7454 version_label: Some("archive".to_string()),
7455 version_note: Some(prepared.reason.clone()),
7456 is_latest: None,
7457 doi: None,
7458 created_at: Some(created_at),
7459 created_by: Some(prepared.actor.clone()),
7460 })
7461 .await
7462 .map_err(|error| core_error("create Archive-study archive asset version", error))?;
7463
7464 let mut bundle_cursor = Cursor::new(bundle_bytes);
7465 let staged = DuckLakeAdapter::new(&session.runtime().lake.data_path)
7466 .stage_datafile_stream(
7467 &StageDataFileStreamRequest {
7468 source_file_name: prepared.archive_datafile.bundle_file_name.clone(),
7469 asset_id: archive_asset.asset_id,
7470 version_id: archive_version.version_id,
7471 study_id: prepared.archive_study.study_id,
7472 asset_name: archive_asset.name.clone(),
7473 major: archive_version.major,
7474 minor: archive_version.minor,
7475 patch: archive_version.patch,
7476 edam_format: prepared.archive_datafile.mime_type.clone(),
7477 compression: DataFileCompression::None,
7478 encryption: DataFileEncryption::None,
7479 },
7480 &mut bundle_cursor,
7481 )
7482 .map_err(|error| {
7483 infrastructure_error("stage Archive-study archive bundle", error)
7484 })?;
7485 let staged_path = datafile_storage_uri_path(
7486 &session.runtime().lake.data_path,
7487 &staged.datafile.storage_uri,
7488 )
7489 .unwrap_or_else(|_| {
7490 Path::new(&session.runtime().lake.data_path).join(&staged.storage_relative_path)
7491 });
7492 Self::bind_staged_datafile(
7493 session.governance_maintenance(),
7494 session.runtime().lake.data_path.clone(),
7495 archive_asset.study_id,
7496 archive_asset.asset_id,
7497 staged.datafile.clone(),
7498 )
7499 .await?;
7500 let archive_datafile = match self.assets.save_datafile(&staged.datafile).await {
7501 Ok(datafile) => datafile,
7502 Err(error) => {
7503 let _ = fs::remove_file(&staged_path);
7504 return Err(core_error("save Archive-study archive datafile", error));
7505 }
7506 };
7507 let _ = ahri_tre_core::record_ingest_provenance(
7508 self.transformations.as_ref(),
7509 &DefaultProvenancePolicy,
7510 &archive_stage_transformation(
7511 "ingest study archive bundle",
7512 &prepared.actor,
7513 ahri_tre_types::TransformationType::Ingest,
7514 ),
7515 &[archive_version.version_id],
7516 )
7517 .await
7518 .map_err(|error| core_error("record Archive-study archive ingest provenance", error))?;
7519
7520 Ok::<_, AppError>(ArchiveStudyForDeletionResult {
7521 prepared,
7522 archive_asset,
7523 archive_version,
7524 archive_datafile,
7525 manifest,
7526 source_size_bytes: staged.source_size_bytes,
7527 stored_size_bytes: staged.staged_size_bytes,
7528 source_study_delete_started: false,
7529 })
7530 }
7531 .await;
7532
7533 let _ = fs::remove_dir_all(&temp_dir);
7534 archive_result
7535 }
7536
7537 async fn authorize_archive_delete_plan(
7538 &self,
7539 target: &DeleteTarget,
7540 actor: &str,
7541 ) -> Result<ArchiveDeleteAuthorization, ArchiveDeleteValidationError> {
7542 match target {
7543 DeleteTarget::Study(study_id) => {
7544 let custodians = self
7545 .study_governance
7546 .list_custodians(*study_id)
7547 .await
7548 .map_err(|error| {
7549 archive_delete_validation_error(
7550 "authorization_unavailable",
7551 format!(
7552 "study deletion authorization could not be evaluated: {}",
7553 redact_archive_delete_message(&error.to_string())
7554 ),
7555 )
7556 })?;
7557
7558 let context = GovernanceContext {
7559 principal: actor.to_string(),
7560 requested_access: AccessMode::Write,
7561 grants: Vec::new(),
7562 custodians,
7563 };
7564 let authorized = DefaultStudyGovernancePolicy
7565 .principal_can_access(&context)
7566 .map_err(|error| {
7567 archive_delete_validation_error(
7568 "authorization_unavailable",
7569 format!(
7570 "study deletion authorization could not be evaluated: {}",
7571 redact_archive_delete_message(&error.to_string())
7572 ),
7573 )
7574 })?;
7575
7576 if !authorized {
7577 return Err(archive_delete_validation_error(
7578 "unauthorized_actor",
7579 "archive/delete planning requires study custodianship or administrator capability",
7580 ));
7581 }
7582
7583 Ok(ArchiveDeleteAuthorization {
7584 status: ArchiveDeleteAuthorizationStatus::Authorized,
7585 principal: actor.to_string(),
7586 requirement: "study custodianship or administrator capability".to_string(),
7587 })
7588 }
7589 DeleteTarget::Asset(_) | DeleteTarget::AssetVersion(_) => {
7590 let study_id = match target {
7591 DeleteTarget::Asset(asset_id) => self
7592 .assets
7593 .get_asset_by_id(*asset_id)
7594 .await
7595 .map_err(|error| {
7596 archive_delete_validation_error(
7597 "authorization_unavailable",
7598 format!(
7599 "study content deletion authorization could not be evaluated: {}",
7600 redact_archive_delete_message(&error.to_string())
7601 ),
7602 )
7603 })?
7604 .ok_or_else(|| {
7605 archive_delete_validation_error(
7606 "target_not_found",
7607 format!("asset not found: {}", asset_id.0),
7608 )
7609 })?
7610 .study_id,
7611 DeleteTarget::AssetVersion(version_id) => self
7612 .catalog_for_version(*version_id)
7613 .await
7614 .map_err(|error| {
7615 archive_delete_validation_error(
7616 "authorization_unavailable",
7617 format!(
7618 "study content deletion authorization could not be evaluated: {}",
7619 redact_archive_delete_message(&error.to_string())
7620 ),
7621 )
7622 })?
7623 .asset
7624 .study_id,
7625 DeleteTarget::Study(_) => unreachable!("study handled above"),
7626 };
7627 self.authorize_study_content_delete(study_id, actor).await
7628 }
7629 }
7630 }
7631
7632 async fn authorize_study_content_delete(
7633 &self,
7634 study_id: StudyId,
7635 actor: &str,
7636 ) -> Result<ArchiveDeleteAuthorization, ArchiveDeleteValidationError> {
7637 let custodians = self
7638 .study_governance
7639 .list_custodians(study_id)
7640 .await
7641 .map_err(|error| {
7642 archive_delete_validation_error(
7643 "authorization_unavailable",
7644 format!(
7645 "study content deletion authorization could not be evaluated: {}",
7646 redact_archive_delete_message(&error.to_string())
7647 ),
7648 )
7649 })?;
7650
7651 let context = GovernanceContext {
7652 principal: actor.to_string(),
7653 requested_access: AccessMode::Write,
7654 grants: Vec::new(),
7655 custodians,
7656 };
7657 let authorized = DefaultStudyGovernancePolicy
7658 .principal_can_manage_study_asset_lifecycle(&context)
7659 .map_err(|error| {
7660 archive_delete_validation_error(
7661 "authorization_unavailable",
7662 format!(
7663 "study content deletion authorization could not be evaluated: {}",
7664 redact_archive_delete_message(&error.to_string())
7665 ),
7666 )
7667 })?;
7668
7669 if !authorized {
7670 return Err(archive_delete_validation_error(
7671 "unauthorized_actor",
7672 "archive/delete planning requires study custodianship or delegate custodianship for dataset/datafile lifecycle operations",
7673 ));
7674 }
7675
7676 Ok(ArchiveDeleteAuthorization {
7677 status: ArchiveDeleteAuthorizationStatus::Authorized,
7678 principal: actor.to_string(),
7679 requirement: "study custodianship or delegate custodianship for dataset/datafile lifecycle operations".to_string(),
7680 })
7681 }
7682
7683 async fn authorize_study_asset_lifecycle_for_app(
7684 &self,
7685 study_id: StudyId,
7686 actor: Option<&str>,
7687 operation: &str,
7688 ) -> Result<(), AppError> {
7689 let actor = actor
7690 .map(str::trim)
7691 .filter(|actor| !actor.is_empty())
7692 .ok_or_else(|| {
7693 AppError::Validation(format!(
7694 "{operation} requires an authenticated TRE user actor"
7695 ))
7696 })?;
7697 let custodians = self
7698 .study_governance
7699 .list_custodians(study_id)
7700 .await
7701 .map_err(|error| core_error("evaluate study asset lifecycle authorization", error))?;
7702 let context = GovernanceContext {
7703 principal: actor.to_string(),
7704 requested_access: AccessMode::Write,
7705 grants: Vec::new(),
7706 custodians,
7707 };
7708 let authorized = DefaultStudyGovernancePolicy
7709 .principal_can_manage_study_asset_lifecycle(&context)
7710 .map_err(|error| core_error("evaluate study asset lifecycle authorization", error))?;
7711 if authorized {
7712 Ok(())
7713 } else {
7714 Err(AppError::Validation(format!(
7715 "{operation} requires study custodianship or delegate custodianship"
7716 )))
7717 }
7718 }
7719
7720 pub async fn list_study_datafiles(
7721 &self,
7722 request: ListStudyDataFilesRequest,
7723 ) -> Result<Vec<StudyDataFileEntry>, AppError> {
7724 let mut assets = self
7725 .list_study_assets(ListStudyAssetsRequest {
7726 study_id: request.study_id,
7727 })
7728 .await?;
7729 assets.retain(|asset| asset.asset_type == ahri_tre_types::AssetType::File);
7730 assets.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str()));
7731
7732 let mut entries = Vec::new();
7733 for asset in assets {
7734 let mut catalog = self.asset_version_catalog(asset).await?;
7735 let selected_versions = if request.include_versions {
7736 catalog.versions.clone()
7737 } else {
7738 catalog.latest_version.iter().cloned().collect()
7739 };
7740 let mut datafiles = Vec::new();
7741 for version in selected_versions {
7742 if let Some(datafile) = self
7743 .assets
7744 .get_datafile(version.version_id)
7745 .await
7746 .map_err(|error| core_error("lookup study datafile", error))?
7747 {
7748 datafiles.push(datafile);
7749 }
7750 }
7751 if !request.include_versions {
7752 catalog.versions.clear();
7753 }
7754 entries.push(StudyDataFileEntry { catalog, datafiles });
7755 }
7756
7757 Ok(entries)
7758 }
7759
7760 pub async fn list_datafiles(
7762 &self,
7763 request: crate::ListDataFilesRequest,
7764 ) -> Result<crate::DataFileListResult, AppError> {
7765 let study = self
7766 .resolve_study_selector(request.study, "catalogue Study")
7767 .await?;
7768 let datafiles = self
7769 .list_study_datafiles(ListStudyDataFilesRequest {
7770 study_id: study.study_id,
7771 include_versions: request.include_versions,
7772 })
7773 .await?;
7774 Ok(crate::DataFileListResult::from_catalogue(study, datafiles))
7775 }
7776
7777 pub async fn get_datafile_metadata(
7778 &self,
7779 request: DataFileMetadataRequest,
7780 ) -> Result<DataFileMetadata, AppError> {
7781 let catalog = self
7782 .get_asset(GetAssetRequest {
7783 asset_id: None,
7784 study_id: Some(request.study_id),
7785 asset_name: Some(request.asset_name),
7786 })
7787 .await?;
7788 if catalog.asset.asset_type != ahri_tre_types::AssetType::File {
7789 return Err(AppError::Validation(format!(
7790 "asset {} is not a file asset",
7791 catalog.asset.name.as_str()
7792 )));
7793 }
7794 let version = resolve_requested_asset_version(&catalog, request.version.as_deref())?;
7795 let datafile = self
7796 .assets
7797 .get_datafile(version.version_id)
7798 .await
7799 .map_err(|error| core_error("lookup datafile metadata", error))?
7800 .ok_or_else(|| {
7801 AppError::Validation(format!(
7802 "datafile metadata not found for version: {}",
7803 version.version_id.0
7804 ))
7805 })?;
7806
7807 Ok(DataFileMetadata {
7808 catalog,
7809 version,
7810 datafile,
7811 })
7812 }
7813
7814 pub async fn get_datafile_metadata_by_id(
7815 &self,
7816 request: DataFileMetadataByIdRequest,
7817 ) -> Result<DataFileMetadata, AppError> {
7818 let catalog = self
7819 .catalog_for_version(request.datafile_id)
7820 .await
7821 .map_err(|error| core_error("lookup datafile asset version", error))?;
7822 if catalog.asset.asset_type != ahri_tre_types::AssetType::File {
7823 return Err(AppError::Validation(format!(
7824 "version {} is not a datafile",
7825 request.datafile_id.0
7826 )));
7827 }
7828 let version = catalog
7829 .versions
7830 .iter()
7831 .find(|version| version.version_id == request.datafile_id)
7832 .cloned()
7833 .ok_or_else(|| {
7834 AppError::Validation(format!(
7835 "asset version not found for datafile metadata: {}",
7836 request.datafile_id.0
7837 ))
7838 })?;
7839 let datafile = self
7840 .assets
7841 .get_datafile(request.datafile_id)
7842 .await
7843 .map_err(|error| core_error("lookup datafile metadata", error))?
7844 .ok_or_else(|| {
7845 AppError::Validation(format!(
7846 "datafile metadata not found for version: {}",
7847 request.datafile_id.0
7848 ))
7849 })?;
7850
7851 Ok(DataFileMetadata {
7852 catalog,
7853 version,
7854 datafile,
7855 })
7856 }
7857
7858 pub async fn record_export(
7859 &self,
7860 request: RecordExportRequest,
7861 ) -> Result<ahri_tre_types::TransformationLineageRecord, AppError> {
7862 let policy = DefaultProvenancePolicy;
7863 let transformation = enrich_transformation_source(&request.transformation);
7864 ahri_tre_core::record_export_provenance(
7865 self.transformations.as_ref(),
7866 &policy,
7867 &transformation,
7868 &request.input_version_ids,
7869 )
7870 .await
7871 .map_err(|error| core_error("record export provenance", error))
7872 }
7873
7874 pub async fn transform_assets(
7875 &self,
7876 request: TransformAssetsRequest,
7877 ) -> Result<ahri_tre_types::TransformationLineageRecord, AppError> {
7878 let policy = DefaultProvenancePolicy;
7879 let transformation = enrich_transformation_source(&request.transformation);
7880 ahri_tre_core::transform_assets(
7881 self.transformations.as_ref(),
7882 &policy,
7883 &transformation,
7884 &request.input_version_ids,
7885 &request.output_version_ids,
7886 )
7887 .await
7888 .map_err(|error| core_error("transform assets provenance", error))
7889 }
7890
7891 async fn dataset_catalog_by_name(
7892 &self,
7893 study_id: ahri_tre_types::StudyId,
7894 dataset_name: ahri_tre_types::NcName,
7895 ) -> Result<AssetVersionCatalog, AppError> {
7896 let asset = self
7897 .assets
7898 .get_asset_by_name(study_id, dataset_name.as_str())
7899 .await
7900 .map_err(|error| core_error("lookup dataset asset by name", error))?
7901 .ok_or_else(|| {
7902 AppError::Validation(format!("dataset not found: {}", dataset_name.as_str()))
7903 })?;
7904 if asset.asset_type != ahri_tre_types::AssetType::Dataset {
7905 return Err(AppError::Validation(format!(
7906 "asset {} is not a dataset",
7907 asset.name.as_str()
7908 )));
7909 }
7910 self.complete_dataset_catalog(asset).await
7911 }
7912
7913 async fn complete_dataset_catalog(
7914 &self,
7915 asset: ahri_tre_types::AssetRecord,
7916 ) -> Result<AssetVersionCatalog, AppError> {
7917 let versions = self
7918 .assets
7919 .list_asset_versions(asset.asset_id)
7920 .await
7921 .map_err(|error| core_error("list dataset asset versions", error))?;
7922 let dataset_ids = self
7923 .assets
7924 .list_datasets_for_asset(asset.asset_id)
7925 .await
7926 .map_err(|error| core_error("list dataset records for asset", error))?
7927 .into_iter()
7928 .map(|dataset| dataset.dataset_id.0)
7929 .collect::<Vec<_>>();
7930 let withdrawn_ids = self
7931 .assets
7932 .list_dataset_version_withdrawals(asset.asset_id)
7933 .await
7934 .map_err(|error| core_error("list dataset version withdrawals", error))?
7935 .into_iter()
7936 .map(|withdrawal| withdrawal.dataset_id.0)
7937 .collect::<Vec<_>>();
7938 let mut complete_versions = Vec::new();
7939 for version in versions {
7940 if !dataset_ids.contains(&version.version_id.0)
7941 || withdrawn_ids.contains(&version.version_id.0)
7942 {
7943 continue;
7944 }
7945 let completed = version.is_latest == Some(true)
7949 || !self
7950 .transformations
7951 .list_transformations_producing(version.version_id)
7952 .await
7953 .map_err(|error| core_error("read Dataset completion provenance", error))?
7954 .is_empty();
7955 if completed {
7956 complete_versions.push(version);
7957 }
7958 }
7959 complete_versions.sort_by_key(|version| (version.major, version.minor, version.patch));
7960 let latest_version = complete_versions
7961 .iter()
7962 .max_by_key(|version| (version.major, version.minor, version.patch))
7963 .cloned()
7964 .map(|mut version| {
7965 version.is_latest = Some(true);
7966 version
7967 });
7968 for version in &mut complete_versions {
7969 version.is_latest = Some(
7970 latest_version
7971 .as_ref()
7972 .is_some_and(|latest| latest.version_id == version.version_id),
7973 );
7974 }
7975
7976 Ok(AssetVersionCatalog {
7977 asset,
7978 versions: complete_versions,
7979 latest_version,
7980 })
7981 }
7982
7983 async fn repair_latest_after_withdrawal(
7984 &self,
7985 asset_id: ahri_tre_types::AssetId,
7986 withdrawn_version_id: ahri_tre_types::VersionId,
7987 was_latest: bool,
7988 ) -> Result<Option<ahri_tre_types::AssetVersionRecord>, AppError> {
7989 if !was_latest {
7990 return Ok(None);
7991 }
7992 let dataset_ids = self
7993 .assets
7994 .list_datasets_for_asset(asset_id)
7995 .await
7996 .map_err(|error| core_error("list datasets for latest repair", error))?
7997 .into_iter()
7998 .map(|dataset| dataset.dataset_id.0)
7999 .collect::<Vec<_>>();
8000 let withdrawn_ids = self
8001 .assets
8002 .list_dataset_version_withdrawals(asset_id)
8003 .await
8004 .map_err(|error| core_error("list withdrawals for latest repair", error))?
8005 .into_iter()
8006 .map(|withdrawal| withdrawal.dataset_id.0)
8007 .collect::<Vec<_>>();
8008 let candidate = self
8009 .assets
8010 .list_asset_versions(asset_id)
8011 .await
8012 .map_err(|error| core_error("list versions for latest repair", error))?
8013 .into_iter()
8014 .filter(|version| {
8015 version.version_id != withdrawn_version_id
8016 && dataset_ids.contains(&version.version_id.0)
8017 && !withdrawn_ids.contains(&version.version_id.0)
8018 })
8019 .max_by_key(|version| (version.major, version.minor, version.patch));
8020 match candidate {
8021 Some(version) => self
8022 .assets
8023 .mark_asset_version_latest(asset_id, version.version_id)
8024 .await
8025 .map(Some)
8026 .map_err(|error| core_error("promote previous dataset version", error)),
8027 None => {
8028 self.assets
8029 .clear_asset_version_latest(asset_id, withdrawn_version_id)
8030 .await
8031 .map_err(|error| core_error("clear withdrawn latest dataset version", error))?;
8032 Ok(None)
8033 }
8034 }
8035 }
8036
8037 async fn datafile_delete_latest_candidate(
8038 &self,
8039 asset_id: ahri_tre_types::AssetId,
8040 deleted_version_id: ahri_tre_types::VersionId,
8041 was_latest: bool,
8042 ) -> Result<Option<ahri_tre_types::AssetVersionRecord>, AppError> {
8043 if !was_latest {
8044 return Ok(None);
8045 }
8046 let datafile_ids = self
8047 .assets
8048 .list_datafiles_for_asset(asset_id)
8049 .await
8050 .map_err(|error| core_error("list datafiles for latest repair", error))?
8051 .into_iter()
8052 .map(|datafile| datafile.datafile_id.0)
8053 .collect::<Vec<_>>();
8054 let candidate = self
8055 .assets
8056 .list_asset_versions(asset_id)
8057 .await
8058 .map_err(|error| core_error("list datafile versions for latest repair", error))?
8059 .into_iter()
8060 .filter(|version| {
8061 version.version_id != deleted_version_id
8062 && datafile_ids.contains(&version.version_id.0)
8063 })
8064 .max_by_key(|version| (version.major, version.minor, version.patch));
8065 Ok(candidate.map(|mut version| {
8066 version.is_latest = Some(true);
8067 version
8068 }))
8069 }
8070
8071 async fn catalog_for_version(
8072 &self,
8073 version_id: ahri_tre_types::VersionId,
8074 ) -> Result<AssetVersionCatalog, CoreError> {
8075 let version = self
8076 .assets
8077 .get_asset_version(version_id)
8078 .await?
8079 .ok_or_else(|| {
8080 CoreError::Validation(format!("asset version not found: {}", version_id.0))
8081 })?;
8082 let asset = self
8083 .assets
8084 .get_asset_by_id(version.asset_id)
8085 .await?
8086 .ok_or_else(|| {
8087 CoreError::Validation(format!("asset not found: {}", version.asset_id.0))
8088 })?;
8089 let versions = self.assets.list_asset_versions(asset.asset_id).await?;
8090 let latest_version = self.latest_catalogue_version(&asset, &versions).await?;
8091 Ok(AssetVersionCatalog {
8092 asset,
8093 versions,
8094 latest_version,
8095 })
8096 }
8097
8098 async fn prepare_dataset_version(
8099 &self,
8100 request: PrepareDatasetVersionRequest,
8101 ) -> Result<PreparedDatasetVersion, AppError> {
8102 let asset_extraction = ahri_tre_core::extract_inline_tags(request.description.as_deref());
8103 let version_extraction =
8104 ahri_tre_core::extract_inline_tags(request.version_note.as_deref());
8105 self.studies
8106 .get_study_by_id(request.study_id)
8107 .await
8108 .map_err(|error| core_error("lookup dataset study", error))?
8109 .ok_or_else(|| {
8110 AppError::Validation(format!("dataset study not found: {}", request.study_id.0))
8111 })?;
8112
8113 let existing_by_name = self
8114 .assets
8115 .get_asset_by_name(request.study_id, request.dataset_name.as_str())
8116 .await
8117 .map_err(|error| core_error("lookup dataset asset by name", error))?;
8118 let existing_asset = match (existing_by_name, request.dataset_asset_id) {
8119 (Some(asset), Some(requested_asset_id)) if asset.asset_id != requested_asset_id => {
8120 return Err(AppError::Validation(format!(
8121 "dataset asset id {} does not match existing asset {} named {}",
8122 requested_asset_id.0,
8123 asset.asset_id.0,
8124 request.dataset_name.as_str()
8125 )));
8126 }
8127 (Some(asset), _) => Some(asset),
8128 (None, Some(requested_asset_id)) => {
8129 let asset = self
8130 .assets
8131 .get_asset_by_id(requested_asset_id)
8132 .await
8133 .map_err(|error| core_error("lookup dataset asset by id", error))?
8134 .ok_or_else(|| {
8135 AppError::Validation(format!(
8136 "requested dataset asset not found: {}",
8137 requested_asset_id.0
8138 ))
8139 })?;
8140 if asset.study_id != request.study_id {
8141 return Err(AppError::Validation(format!(
8142 "dataset asset {} belongs to study {}, not {}",
8143 asset.asset_id.0, asset.study_id.0, request.study_id.0
8144 )));
8145 }
8146 if asset.name != request.dataset_name {
8147 return Err(AppError::Validation(format!(
8148 "dataset asset {} is named {}, not {}",
8149 asset.asset_id.0,
8150 asset.name.as_str(),
8151 request.dataset_name.as_str()
8152 )));
8153 }
8154 Some(asset)
8155 }
8156 (None, None) => None,
8157 };
8158
8159 match (request.output_intent, existing_asset.is_some()) {
8160 (Some(DatasetOutputIntent::Create), true) => {
8161 return Err(AppError::Conflict(
8162 "Dataset output already exists; select replacement or a new version".into(),
8163 ));
8164 }
8165 (Some(DatasetOutputIntent::Replace), false) => {
8166 return Err(AppError::Validation(
8167 "Replacement requires an existing Dataset".into(),
8168 ));
8169 }
8170 _ => {}
8171 }
8172 let (asset, asset_persisted) = match existing_asset {
8173 Some(asset) => {
8174 if asset.asset_type != ahri_tre_types::AssetType::Dataset {
8175 return Err(AppError::Validation(format!(
8176 "dataset asset {} is not a dataset asset",
8177 asset.asset_id.0
8178 )));
8179 }
8180 (asset, true)
8181 }
8182 None => {
8183 let new_asset = ahri_tre_types::NewAssetRecord {
8184 study_id: request.study_id,
8185 name: request.dataset_name,
8186 description: asset_extraction.cleaned_text.clone(),
8187 agent_instructions: request.agent_instructions,
8188 asset_type: ahri_tre_types::AssetType::Dataset,
8189 date_created: None,
8190 created_by: request.created_by.clone(),
8191 };
8192 (
8193 ahri_tre_types::AssetRecord {
8194 asset_id: request
8195 .dataset_asset_id
8196 .unwrap_or_else(|| ahri_tre_types::AssetId(uuid::Uuid::new_v4())),
8197 study_id: new_asset.study_id,
8198 name: new_asset.name,
8199 description: new_asset.description,
8200 agent_instructions: new_asset.agent_instructions,
8201 asset_type: new_asset.asset_type,
8202 date_created: new_asset.date_created,
8203 created_by: new_asset.created_by,
8204 },
8205 false,
8206 )
8207 }
8208 };
8209
8210 let existing_versions = if !asset_persisted {
8211 Vec::new()
8212 } else {
8213 self.assets
8214 .list_asset_versions(asset.asset_id)
8215 .await
8216 .map_err(|error| core_error("list dataset asset versions", error))?
8217 };
8218 let (major, minor, patch) = match request.output_intent {
8219 Some(DatasetOutputIntent::Version(major, minor, patch)) => (major, minor, patch),
8220 _ => ahri_tre_core::next_asset_version_coordinates(&existing_versions)
8221 .map_err(|error| core_error("choose dataset asset version", error))?,
8222 };
8223 let (version, version_persisted) = if let Some(version_id) = request.dataset_version_id {
8224 let mut version = ahri_tre_types::AssetVersionRecord {
8225 version_id,
8226 asset_id: asset.asset_id,
8227 major,
8228 minor,
8229 patch,
8230 version_label: None,
8231 version_note: version_extraction.cleaned_text.clone(),
8232 is_latest: None,
8233 doi: None,
8234 created_at: None,
8235 created_by: request.created_by,
8236 };
8237 ahri_tre_core::validate_new_asset_version(&existing_versions, &version)
8238 .map_err(|error| core_error("validate dataset asset version", error))?;
8239 version.is_latest = Some(false);
8240 (version, false)
8241 } else {
8242 let version = ahri_tre_types::NewAssetVersionRecord {
8243 classification: request.classification.clone(),
8244 asset_id: asset.asset_id,
8245 major,
8246 minor,
8247 patch,
8248 version_label: None,
8249 version_note: version_extraction.cleaned_text.clone(),
8250 is_latest: None,
8251 doi: None,
8252 created_at: None,
8253 created_by: request.created_by,
8254 };
8255 ahri_tre_core::validate_new_asset_version_input(&existing_versions, &version)
8256 .map_err(|error| core_error("validate dataset asset version", error))?;
8257 let version = ahri_tre_core::prepare_new_asset_version_for_create(version)
8258 .map_err(|error| core_error("prepare dataset asset version", error))?;
8259 (
8260 ahri_tre_types::AssetVersionRecord {
8261 version_id: ahri_tre_types::VersionId(uuid::Uuid::new_v4()),
8262 asset_id: version.asset_id,
8263 major: version.major,
8264 minor: version.minor,
8265 patch: version.patch,
8266 version_label: version.version_label,
8267 version_note: version.version_note,
8268 is_latest: Some(false),
8269 doi: version.doi,
8270 created_at: version.created_at,
8271 created_by: version.created_by,
8272 },
8273 false,
8274 )
8275 };
8276
8277 Ok(PreparedDatasetVersion {
8278 ordinary_upload_source: None,
8279 inherit_risk: request.inherit_risk,
8280 classification: request.classification.clone(),
8281 asset,
8282 version,
8283 asset_persisted,
8284 version_persisted,
8285 asset_tags: asset_extraction.tags,
8286 version_tags: version_extraction.tags,
8287 })
8288 }
8289
8290 async fn save_dataset_metadata_with_options(
8291 &self,
8292 session: &DataStoreSession,
8293 attempt: &mut DatasetMaterializationAttempt,
8294 dataset: ahri_tre_types::DatasetRecord,
8295 transformation: &ahri_tre_types::NewTransformationRecord,
8296 input_version_ids: &[ahri_tre_types::VersionId],
8297 registration: DatasetMetadataRegistration<'_>,
8298 ) -> Result<DatasetAdmissionRecords, AppError> {
8299 self.save_dataset_metadata_with_completion(
8300 session,
8301 attempt,
8302 DatasetAdmissionMetadata {
8303 dataset,
8304 transformation,
8305 input_version_ids,
8306 registration,
8307 },
8308 |_, _| Ok(()),
8309 )
8310 .await
8311 }
8312
8313 async fn save_dataset_metadata_with_completion(
8316 &self,
8317 session: &DataStoreSession,
8318 attempt: &mut DatasetMaterializationAttempt,
8319 metadata: DatasetAdmissionMetadata<'_>,
8320 completion: impl FnOnce(
8321 &ahri_tre_pgmeta::PgMetadataRepository<'_>,
8322 &DatasetAdmissionRecords,
8323 ) -> Result<(), AppError>,
8324 ) -> Result<DatasetAdmissionRecords, AppError> {
8325 let DatasetAdmissionMetadata {
8326 dataset,
8327 transformation,
8328 input_version_ids,
8329 registration,
8330 } = metadata;
8331 if session.disclosure_context.is_some() {
8332 let prepared = &attempt.prepared;
8333 let (table_uuid, lake_snapshot) = ahri_tre_lake::dataset_physical_identity(
8334 &session.lake,
8335 session
8336 .runtime()
8337 .lake
8338 .catalog_schema
8339 .as_deref()
8340 .ok_or_else(|| {
8341 AppError::Infrastructure("Dataset catalog identity is unavailable".into())
8342 })?,
8343 prepared.asset.study_id,
8344 prepared.asset.name.as_str(),
8345 prepared.version.major,
8346 prepared.version.minor,
8347 prepared.version.patch,
8348 )
8349 .map_err(|_| {
8350 AppError::Infrastructure("Dataset physical identity is unavailable".into())
8351 })?;
8352 session
8353 .governance_maintenance()
8354 .ok_or_else(|| {
8355 AppError::Infrastructure("Dataset governance authority is unavailable".into())
8356 })?
8357 .bind_dataset(&ahri_tre_types::GovernedDatasetLocation {
8358 version_id: prepared.version.version_id.0,
8359 asset_id: prepared.asset.asset_id.0,
8360 study_id: prepared.asset.study_id.0,
8361 name: prepared.asset.name.clone(),
8362 major: prepared.version.major,
8363 minor: prepared.version.minor,
8364 patch: prepared.version.patch,
8365 table_uuid,
8366 lake_snapshot,
8367 })
8368 .map_err(|_| {
8369 AppError::Infrastructure("Dataset origin persistence is unavailable".into())
8370 })?;
8371 }
8372 attempt
8373 .scratch
8374 .cleanup()
8375 .map_err(|e| infrastructure_error("clean Dataset attempt source", e))?;
8376 let prepared = &attempt.prepared;
8377 #[allow(irrefutable_let_patterns)]
8378 if let WriterAuthority::Postgres(reservation) = &mut attempt.authority {
8379 let repository = session.dataset_admission_repository().ok_or_else(|| {
8380 AppError::Infrastructure("Dataset metadata admission is unavailable".into())
8381 })?;
8382 let repository = match attempt.observation {
8383 Some(context) => repository.with_correlation(context),
8384 None => repository,
8385 };
8386 return repository
8387 .with_reserved_dataset_admission(
8388 reservation,
8389 &prepared.asset,
8390 &prepared.version,
8391 |repository| {
8392 let repository = Arc::new(repository);
8393 let repositories =
8394 crate::session::ScopedSessionMetadataRepositories::from_repository(
8395 Arc::clone(&repository),
8396 );
8397 let mut service = ScopedAppService::new(repositories.into());
8398 service.semantic_repository = Some((*repository).clone());
8399 futures::FutureExt::now_or_never(
8402 service.write_dataset_metadata_in_admission(
8403 prepared,
8404 dataset,
8405 transformation,
8406 input_version_ids,
8407 registration,
8408 ),
8409 )
8410 .unwrap_or_else(|| {
8411 Err(AppError::Infrastructure(
8412 "Dataset metadata admission unexpectedly suspended".into(),
8413 ))
8414 })
8415 .and_then(|records| {
8416 completion(&repository, &records)?;
8417 Ok(records)
8418 })
8419 },
8420 )
8421 .map_err(|error| match error {
8422 ahri_tre_pgmeta::DatasetAdmissionError::Rejected(error) => error,
8423 ahri_tre_pgmeta::DatasetAdmissionError::Metadata(error) => {
8424 core_error("admit Dataset", error)
8425 }
8426 ahri_tre_pgmeta::DatasetAdmissionError::OutcomeUnknown => {
8427 AppError::DatasetAdmissionOutcomeUnknown(prepared.version.version_id)
8428 }
8429 });
8430 }
8431 #[cfg(test)]
8434 return self
8435 .write_dataset_metadata_in_admission(
8436 prepared,
8437 dataset,
8438 transformation,
8439 input_version_ids,
8440 registration,
8441 )
8442 .await;
8443 #[cfg(not(test))]
8444 Err(AppError::Infrastructure(
8445 "Dataset metadata admission is unavailable".into(),
8446 ))
8447 }
8448
8449 async fn registered_variable_metadata(
8450 &self,
8451 variable: ahri_tre_types::VariableRecord,
8452 ) -> Result<RegisteredVariable, AppError> {
8453 let value_type = self.value_type_for_id(variable.value_type_id).await?;
8454 let (vocabulary, vocabulary_items) = self.variable_vocabulary_metadata(&variable).await?;
8455 Ok(RegisteredVariable {
8456 variable,
8457 value_type,
8458 vocabulary,
8459 vocabulary_items,
8460 })
8461 }
8462
8463 async fn registered_vocabulary_metadata(
8464 &self,
8465 vocabulary: ahri_tre_types::VocabularyRecord,
8466 ) -> Result<RegisteredVocabulary, AppError> {
8467 let items = if self.semantic_repository.is_some() {
8468 Vec::new()
8471 } else {
8472 self.vocabularies
8473 .list_vocabulary_items(vocabulary.vocabulary_id)
8474 .await
8475 .map_err(|error| core_error("list vocabulary items", error))?
8476 };
8477 Ok(RegisteredVocabulary { vocabulary, items })
8478 }
8479
8480 async fn add_or_update_variable(
8481 &self,
8482 request: VariableCommandFields,
8483 ) -> Result<RegisteredVariable, AppError> {
8484 let VariableCommandFields {
8485 domain,
8486 name,
8487 value_type,
8488 value_format,
8489 key_role,
8490 description,
8491 ontology_namespace,
8492 ontology_class,
8493 vocabulary,
8494 vocabulary_items,
8495 existing,
8496 force_metadata_updates,
8497 } = request;
8498 let domain = self
8499 .resolve_domain_selector(domain, "variable domain")
8500 .await?;
8501 let variable_name = parse_variable_name(name)?;
8502 let vocabulary_id = match vocabulary {
8503 Some(vocabulary) => Some(
8504 self.resolve_vocabulary_reference_in_domain(domain.domain_id, vocabulary.as_str())
8505 .await?
8506 .vocabulary_id,
8507 ),
8508 None => existing
8509 .as_ref()
8510 .and_then(|variable| variable.vocabulary_id),
8511 };
8512 if let Some(items) = vocabulary_items.as_deref() {
8513 let vocabulary_id = vocabulary_id.ok_or_else(|| {
8514 AppError::Validation(
8515 "categorical variable item validation requires a vocabulary".to_string(),
8516 )
8517 })?;
8518 self.validate_variable_vocabulary_items(vocabulary_id, items)
8519 .await?;
8520 }
8521
8522 let value_type_name = value_type
8523 .as_deref()
8524 .or_else(|| existing.as_ref().map(|_| ""))
8525 .unwrap_or("");
8526 if is_enumeration_value_type(value_type_name) && vocabulary_id.is_none() {
8527 return Err(AppError::Validation(
8528 "categorical variables require a governed vocabulary".to_string(),
8529 ));
8530 }
8531
8532 let existing_value_type_id = existing.as_ref().map(|variable| variable.value_type_id);
8533 let existing_key_role = existing
8534 .as_ref()
8535 .map(|variable| variable.key_role)
8536 .unwrap_or(ahri_tre_types::KeyRole::None);
8537 self.register_variable(RegisterVariableRequest {
8538 domain_id: domain.domain_id,
8539 variable_id: existing.as_ref().map(|variable| variable.variable_id),
8540 name: variable_name,
8541 value_type_id: if value_type.is_some() {
8542 None
8543 } else {
8544 existing_value_type_id
8545 },
8546 value_type,
8547 value_format: value_format.or_else(|| {
8548 existing
8549 .as_ref()
8550 .and_then(|variable| variable.value_format.clone())
8551 }),
8552 vocabulary_id,
8553 key_role: match key_role {
8554 Some(key_role) => parse_key_role(&key_role)?,
8555 None => existing_key_role,
8556 },
8557 description: description.or_else(|| {
8558 existing
8559 .as_ref()
8560 .and_then(|variable| variable.description.clone())
8561 }),
8562 note: existing.as_ref().and_then(|variable| variable.note.clone()),
8563 ontology_namespace: ontology_namespace.or_else(|| {
8564 existing
8565 .as_ref()
8566 .and_then(|variable| variable.ontology_namespace.clone())
8567 }),
8568 ontology_class: ontology_class.or_else(|| {
8569 existing
8570 .as_ref()
8571 .and_then(|variable| variable.ontology_class.clone())
8572 }),
8573 force_metadata_updates,
8574 })
8575 .await
8576 }
8577
8578 async fn resolve_semantic_delete_plan(
8579 &self,
8580 target: &SemanticDeleteTarget,
8581 ) -> Result<
8582 Option<(
8583 SemanticDeleteTargetIdentity,
8584 SemanticDeleteDependencySummary,
8585 )>,
8586 AppError,
8587 > {
8588 match target {
8589 SemanticDeleteTarget::Domain { domain } => {
8590 let Some(domain) = self
8591 .get_domain(GetDomainRequest {
8592 domain: domain.clone(),
8593 })
8594 .await?
8595 else {
8596 return Ok(None);
8597 };
8598 let mut summary = SemanticDeleteDependencySummary::clear(semantic_dependency(
8599 "domain",
8600 domain.domain_id.0,
8601 format!(
8602 "domain {} is a physical delete candidate",
8603 domain.name.as_str()
8604 ),
8605 ));
8606 for study in self
8607 .studies
8608 .list_studies()
8609 .await
8610 .map_err(|error| core_error("list studies", error))?
8611 {
8612 let domains = self
8613 .study_domains
8614 .list_domains_for_study(study.study_id)
8615 .await
8616 .map_err(|error| core_error("list study domains", error))?;
8617 if domains
8618 .iter()
8619 .any(|candidate| candidate.domain_id == domain.domain_id)
8620 {
8621 summary.active_blockers.push(semantic_dependency(
8622 "study_domain",
8623 study.study_id.0,
8624 format!("study {} belongs to the domain", study.name.as_str()),
8625 ));
8626 }
8627 }
8628 self.push_domain_semantic_blockers(domain.domain_id, &mut summary)
8629 .await?;
8630 finalize_semantic_summary(&mut summary);
8631 Ok(Some((
8632 semantic_identity(
8633 SemanticDeleteTargetKind::Domain,
8634 domain.domain_id,
8635 domain.domain_id.0,
8636 None,
8637 domain.name.as_str(),
8638 ),
8639 summary,
8640 )))
8641 }
8642 SemanticDeleteTarget::Variable { variable } => {
8643 let Some(registered) = self
8644 .get_variable(GetVariableRequest {
8645 variable: variable.clone(),
8646 })
8647 .await?
8648 else {
8649 return Ok(None);
8650 };
8651 let variable = registered.variable;
8652 let domain_name = self
8653 .domains
8654 .get_domain_by_id(variable.domain_id)
8655 .await
8656 .map_err(|error| core_error("lookup variable domain", error))?
8657 .map(|domain| domain.name.as_str().to_string());
8658 let mut summary = SemanticDeleteDependencySummary::clear(semantic_dependency(
8659 "variable",
8660 variable.variable_id.0,
8661 format!(
8662 "variable {} is a physical delete candidate",
8663 variable.name.as_str()
8664 ),
8665 ));
8666 for link in self
8667 .variables
8668 .list_dataset_variables_for_variable(variable.variable_id)
8669 .await
8670 .map_err(|error| core_error("list dataset-variable links", error))?
8671 {
8672 let action = match self
8673 .assets
8674 .get_asset_version(link.dataset_id)
8675 .await
8676 .map_err(|error| core_error("lookup dataset-variable asset version", error))?
8677 {
8678 Some(version) => match self
8679 .assets
8680 .get_asset_by_id(version.asset_id)
8681 .await
8682 .map_err(|error| core_error("lookup dataset-variable asset", error))?
8683 {
8684 Some(asset) => format!(
8685 "dataset-variable metadata references the variable in dataset {} {}",
8686 asset.name.as_str(),
8687 asset_version_label_text(&version)
8688 ),
8689 None => "dataset-variable metadata references the variable in an unknown dataset asset"
8690 .to_string(),
8691 },
8692 None => {
8693 "dataset-variable metadata references the variable in an unknown dataset version"
8694 .to_string()
8695 }
8696 };
8697 summary.historical_blockers.push(semantic_dependency(
8698 "dataset_variable",
8699 link.dataset_id.0,
8700 action,
8701 ));
8702 }
8703 finalize_semantic_summary(&mut summary);
8704 Ok(Some((
8705 semantic_identity(
8706 SemanticDeleteTargetKind::Variable,
8707 variable.domain_id,
8708 variable.variable_id.0,
8709 domain_name,
8710 variable.name.as_str(),
8711 ),
8712 summary,
8713 )))
8714 }
8715 SemanticDeleteTarget::Vocabulary { vocabulary } => {
8716 let Some(registered) = self
8717 .get_vocabulary(GetVocabularyRequest {
8718 vocabulary: vocabulary.clone(),
8719 })
8720 .await?
8721 else {
8722 return Ok(None);
8723 };
8724 let vocabulary = registered.vocabulary;
8725 let domain_name = self
8726 .domains
8727 .get_domain_by_id(vocabulary.domain_id)
8728 .await
8729 .map_err(|error| core_error("lookup vocabulary domain", error))?
8730 .map(|domain| domain.name.as_str().to_string());
8731 let mut summary = SemanticDeleteDependencySummary::clear(semantic_dependency(
8732 "vocabulary",
8733 vocabulary.vocabulary_id.0,
8734 format!(
8735 "vocabulary {} is a physical delete candidate",
8736 vocabulary.name.as_str()
8737 ),
8738 ));
8739 for variable in self
8740 .variables
8741 .list_domain_variables(vocabulary.domain_id)
8742 .await
8743 .map_err(|error| core_error("list domain variables", error))?
8744 {
8745 if variable.vocabulary_id == Some(vocabulary.vocabulary_id) {
8746 summary.active_blockers.push(semantic_dependency(
8747 "variable_vocabulary",
8748 variable.variable_id.0,
8749 format!("variable {} uses the vocabulary", variable.name.as_str()),
8750 ));
8751 }
8752 }
8753 let items = self
8754 .vocabularies
8755 .list_vocabulary_items(vocabulary.vocabulary_id)
8756 .await
8757 .map_err(|error| core_error("list vocabulary items", error))?;
8758 let item_ids = items
8759 .iter()
8760 .map(|item| item.vocabulary_item_id)
8761 .collect::<Vec<_>>();
8762 for item in items {
8763 summary.active_blockers.push(semantic_dependency(
8764 "vocabulary_item",
8765 item.vocabulary_item_id.0,
8766 format!("vocabulary item {} belongs to the vocabulary", item.code),
8767 ));
8768 }
8769 for mapping in self
8770 .vocabularies
8771 .list_vocabulary_mappings()
8772 .await
8773 .map_err(|error| core_error("list vocabulary mappings", error))?
8774 {
8775 if item_ids.contains(&mapping.from_vocabulary_item)
8776 || item_ids.contains(&mapping.to_vocabulary_item)
8777 {
8778 summary.historical_blockers.push(semantic_dependency(
8779 "vocabulary_mapping",
8780 mapping.vocabulary_mapping_id,
8781 "vocabulary mapping references an item in the vocabulary",
8782 ));
8783 }
8784 }
8785 finalize_semantic_summary(&mut summary);
8786 Ok(Some((
8787 semantic_identity(
8788 SemanticDeleteTargetKind::Vocabulary,
8789 vocabulary.domain_id,
8790 vocabulary.vocabulary_id.0,
8791 domain_name,
8792 vocabulary.name.as_str(),
8793 ),
8794 summary,
8795 )))
8796 }
8797 SemanticDeleteTarget::Entity { entity } => {
8798 let Some(entity) = self
8799 .get_entity(GetEntityRequest {
8800 entity: entity.clone(),
8801 })
8802 .await?
8803 else {
8804 return Ok(None);
8805 };
8806 let domain_name = self
8807 .domains
8808 .get_domain_by_id(entity.domain_id)
8809 .await
8810 .map_err(|error| core_error("lookup entity domain", error))?
8811 .map(|domain| domain.name.as_str().to_string());
8812 let mut summary = SemanticDeleteDependencySummary::clear(semantic_dependency(
8813 "entity",
8814 entity.entity_id.0,
8815 format!(
8816 "entity {} is a physical delete candidate",
8817 entity.name.as_str()
8818 ),
8819 ));
8820 for relation in self
8821 .entities
8822 .list_entity_relations(entity.domain_id)
8823 .await
8824 .map_err(|error| core_error("list entity relations", error))?
8825 {
8826 if relation.subject_entity_id == entity.entity_id
8827 || relation.object_entity_id == entity.entity_id
8828 {
8829 summary.active_blockers.push(semantic_dependency(
8830 "entity_relation_definition",
8831 relation.entity_relation_id.0,
8832 format!("relation {} uses the entity", relation.name.as_str()),
8833 ));
8834 }
8835 }
8836 for instance in self
8837 .entities
8838 .list_entity_instances(entity.entity_id)
8839 .await
8840 .map_err(|error| core_error("list entity instances", error))?
8841 {
8842 summary.historical_blockers.push(semantic_dependency(
8843 "entity_instance",
8844 instance.instance_id,
8845 "entity instance history references the entity",
8846 ));
8847 }
8848 finalize_semantic_summary(&mut summary);
8849 Ok(Some((
8850 semantic_identity(
8851 SemanticDeleteTargetKind::Entity,
8852 entity.domain_id,
8853 entity.entity_id.0,
8854 domain_name,
8855 entity.name.as_str(),
8856 ),
8857 summary,
8858 )))
8859 }
8860 SemanticDeleteTarget::EntityRelation { relation } => {
8861 let Some(relation) = self
8862 .get_entity_relation(GetEntityRelationRequest {
8863 relation: relation.clone(),
8864 })
8865 .await?
8866 else {
8867 return Ok(None);
8868 };
8869 let domain_name = self
8870 .domains
8871 .get_domain_by_id(relation.domain_id)
8872 .await
8873 .map_err(|error| core_error("lookup relation domain", error))?
8874 .map(|domain| domain.name.as_str().to_string());
8875 let mut summary = SemanticDeleteDependencySummary::clear(semantic_dependency(
8876 "entity_relation",
8877 relation.entity_relation_id.0,
8878 format!(
8879 "entity relation {} is a physical delete candidate",
8880 relation.name.as_str()
8881 ),
8882 ));
8883 for instance in self
8884 .entities
8885 .list_relation_instances(relation.entity_relation_id)
8886 .await
8887 .map_err(|error| core_error("list relation instances", error))?
8888 {
8889 summary.historical_blockers.push(semantic_dependency(
8890 "relation_instance",
8891 instance.relation_instance_id,
8892 "relation instance history references the relation",
8893 ));
8894 }
8895 finalize_semantic_summary(&mut summary);
8896 Ok(Some((
8897 semantic_identity(
8898 SemanticDeleteTargetKind::EntityRelation,
8899 relation.domain_id,
8900 relation.entity_relation_id.0,
8901 domain_name,
8902 relation.name.as_str(),
8903 ),
8904 summary,
8905 )))
8906 }
8907 }
8908 }
8909
8910 async fn push_domain_semantic_blockers(
8911 &self,
8912 domain_id: ahri_tre_types::DomainId,
8913 summary: &mut SemanticDeleteDependencySummary,
8914 ) -> Result<(), AppError> {
8915 for variable in self
8916 .variables
8917 .list_domain_variables(domain_id)
8918 .await
8919 .map_err(|error| core_error("list domain variables", error))?
8920 {
8921 summary.active_blockers.push(semantic_dependency(
8922 "variable",
8923 variable.variable_id.0,
8924 format!("variable {} belongs to the domain", variable.name.as_str()),
8925 ));
8926 }
8927 for vocabulary in self
8928 .vocabularies
8929 .list_vocabularies(domain_id)
8930 .await
8931 .map_err(|error| core_error("list domain vocabularies", error))?
8932 {
8933 summary.active_blockers.push(semantic_dependency(
8934 "vocabulary",
8935 vocabulary.vocabulary_id.0,
8936 format!(
8937 "vocabulary {} belongs to the domain",
8938 vocabulary.name.as_str()
8939 ),
8940 ));
8941 }
8942 for entity in self
8943 .entities
8944 .list_entities(domain_id)
8945 .await
8946 .map_err(|error| core_error("list domain entities", error))?
8947 {
8948 summary.active_blockers.push(semantic_dependency(
8949 "entity",
8950 entity.entity_id.0,
8951 format!("entity {} belongs to the domain", entity.name.as_str()),
8952 ));
8953 }
8954 for relation in self
8955 .entities
8956 .list_entity_relations(domain_id)
8957 .await
8958 .map_err(|error| core_error("list domain entity relations", error))?
8959 {
8960 summary.active_blockers.push(semantic_dependency(
8961 "entity_relation",
8962 relation.entity_relation_id.0,
8963 format!(
8964 "entity relation {} belongs to the domain",
8965 relation.name.as_str()
8966 ),
8967 ));
8968 }
8969 Ok(())
8970 }
8971
8972 async fn resolve_vocabulary_reference_in_domain(
8973 &self,
8974 domain_id: ahri_tre_types::DomainId,
8975 reference: &str,
8976 ) -> Result<ahri_tre_types::VocabularyRecord, AppError> {
8977 let vocabularies = self
8978 .vocabularies
8979 .list_vocabularies(domain_id)
8980 .await
8981 .map_err(|error| core_error("list domain vocabularies", error))?;
8982 if let Ok(id) = reference.parse::<i64>() {
8983 return vocabularies
8984 .into_iter()
8985 .find(|vocabulary| vocabulary.vocabulary_id == ahri_tre_types::VocabularyId(id))
8986 .ok_or_else(|| {
8987 AppError::Validation(format!(
8988 "vocabulary {reference} not found in domain {}",
8989 domain_id.0
8990 ))
8991 });
8992 }
8993 let name = parse_vocabulary_name(reference.to_string())?;
8994 vocabularies
8995 .into_iter()
8996 .find(|vocabulary| vocabulary.name == name)
8997 .ok_or_else(|| {
8998 AppError::Validation(format!(
8999 "vocabulary {reference} not found in domain {}",
9000 domain_id.0
9001 ))
9002 })
9003 }
9004
9005 async fn validate_variable_vocabulary_items(
9006 &self,
9007 vocabulary_id: ahri_tre_types::VocabularyId,
9008 expected_items: &str,
9009 ) -> Result<(), AppError> {
9010 let items = self
9011 .vocabularies
9012 .list_vocabulary_items(vocabulary_id)
9013 .await
9014 .map_err(|error| core_error("list variable vocabulary items", error))?;
9015 for expected in split_cli_list(expected_items) {
9016 let found = items
9017 .iter()
9018 .any(|item| item.code == expected || item.value.to_string() == expected);
9019 if !found {
9020 return Err(AppError::Validation(format!(
9021 "vocabulary item {expected} not found in vocabulary {}",
9022 vocabulary_id.0
9023 )));
9024 }
9025 }
9026 Ok(())
9027 }
9028
9029 async fn dataset_variable_metadata(
9030 &self,
9031 link: ahri_tre_types::DatasetVariableLinkRecord,
9032 ) -> Result<DatasetVariableMetadata, AppError> {
9033 let variable = self
9034 .variables
9035 .get_variable(link.variable_id)
9036 .await
9037 .map_err(|error| core_error("lookup dataset variable", error))?
9038 .ok_or_else(|| {
9039 AppError::Validation(format!("dataset variable {} not found", link.variable_id.0))
9040 })?;
9041 let value_type = self.value_type_for_id(variable.value_type_id).await?;
9042 let (vocabulary, vocabulary_items) = self.variable_vocabulary_metadata(&variable).await?;
9043 Ok(DatasetVariableMetadata {
9044 link,
9045 variable,
9046 value_type,
9047 vocabulary,
9048 vocabulary_items,
9049 })
9050 }
9051
9052 async fn variable_vocabulary_metadata(
9053 &self,
9054 variable: &ahri_tre_types::VariableRecord,
9055 ) -> Result<
9056 (
9057 Option<ahri_tre_types::VocabularyRecord>,
9058 Vec<ahri_tre_types::VocabularyItemRecord>,
9059 ),
9060 AppError,
9061 > {
9062 let Some(vocabulary_id) = variable.vocabulary_id else {
9063 return Ok((None, Vec::new()));
9064 };
9065 let vocabulary = self
9066 .vocabularies
9067 .list_vocabularies(variable.domain_id)
9068 .await
9069 .map_err(|error| core_error("list variable vocabularies", error))?
9070 .into_iter()
9071 .find(|vocabulary| vocabulary.vocabulary_id == vocabulary_id)
9072 .ok_or_else(|| {
9073 AppError::Validation(format!(
9074 "variable {} references missing vocabulary {}",
9075 variable.name.as_str(),
9076 vocabulary_id.0
9077 ))
9078 })?;
9079 let items = if self.semantic_repository.is_some() {
9080 Vec::new()
9083 } else {
9084 self.vocabularies
9085 .list_vocabulary_items(vocabulary.vocabulary_id)
9086 .await
9087 .map_err(|error| core_error("list variable vocabulary items", error))?
9088 };
9089 Ok((Some(vocabulary), items))
9090 }
9091
9092 async fn vocabulary_mapping_detail(
9093 &self,
9094 mapping: ahri_tre_types::VocabularyMappingRecord,
9095 ) -> Result<VocabularyMappingDetail, AppError> {
9096 let from_item = self
9097 .vocabularies
9098 .get_vocabulary_item(mapping.from_vocabulary_item)
9099 .await
9100 .map_err(|error| core_error("lookup mapping source item", error))?
9101 .ok_or_else(|| {
9102 AppError::Validation(format!(
9103 "mapping {} source item {} not found",
9104 mapping.vocabulary_mapping_id, mapping.from_vocabulary_item.0
9105 ))
9106 })?;
9107 let to_item = self
9108 .vocabularies
9109 .get_vocabulary_item(mapping.to_vocabulary_item)
9110 .await
9111 .map_err(|error| core_error("lookup mapping target item", error))?
9112 .ok_or_else(|| {
9113 AppError::Validation(format!(
9114 "mapping {} target item {} not found",
9115 mapping.vocabulary_mapping_id, mapping.to_vocabulary_item.0
9116 ))
9117 })?;
9118 let from_vocabulary = self
9119 .vocabularies
9120 .get_vocabulary(from_item.vocabulary_id)
9121 .await
9122 .map_err(|error| core_error("lookup mapping source vocabulary", error))?
9123 .ok_or_else(|| {
9124 AppError::Validation(format!(
9125 "mapping source vocabulary {} not found",
9126 from_item.vocabulary_id.0
9127 ))
9128 })?;
9129 let to_vocabulary = self
9130 .vocabularies
9131 .get_vocabulary(to_item.vocabulary_id)
9132 .await
9133 .map_err(|error| core_error("lookup mapping target vocabulary", error))?
9134 .ok_or_else(|| {
9135 AppError::Validation(format!(
9136 "mapping target vocabulary {} not found",
9137 to_item.vocabulary_id.0
9138 ))
9139 })?;
9140 Ok(VocabularyMappingDetail {
9141 mapping,
9142 from_vocabulary,
9143 from_item,
9144 to_vocabulary,
9145 to_item,
9146 })
9147 }
9148
9149 async fn resolve_vocabulary_item_reference(
9150 &self,
9151 label: &str,
9152 reference: &VocabularyItemReference,
9153 ) -> Result<ahri_tre_types::VocabularyItemRecord, AppError> {
9154 if let Some(vocabulary_item_id) = reference.vocabulary_item_id {
9155 let item = self
9156 .vocabularies
9157 .get_vocabulary_item(vocabulary_item_id)
9158 .await
9159 .map_err(|error| core_error("lookup vocabulary item", error))?
9160 .ok_or_else(|| {
9161 AppError::Validation(format!("{label} {} not found", vocabulary_item_id.0))
9162 })?;
9163 self.validate_vocabulary_item_reference_context(label, &item, reference)
9164 .await?;
9165 return Ok(item);
9166 }
9167
9168 let vocabulary_id = self.resolve_vocabulary_reference(label, reference).await?;
9169 let items = self
9170 .vocabularies
9171 .list_vocabulary_items(vocabulary_id)
9172 .await
9173 .map_err(|error| core_error("list vocabulary items", error))?;
9174 let matches: Vec<_> = items
9175 .into_iter()
9176 .filter(|item| {
9177 reference.value.is_none_or(|value| item.value == value)
9178 && reference
9179 .code
9180 .as_ref()
9181 .is_none_or(|code| &item.code == code)
9182 })
9183 .collect();
9184 if reference.value.is_none() && reference.code.is_none() {
9185 return Err(AppError::Validation(format!(
9186 "{label} requires a vocabulary_item_id, value, or code"
9187 )));
9188 }
9189 match matches.len() {
9190 1 => Ok(matches.into_iter().next().expect("one match was present")),
9191 0 => Err(AppError::Validation(format!(
9192 "{label} not found in vocabulary {}",
9193 vocabulary_id.0
9194 ))),
9195 _ => Err(AppError::Validation(format!(
9196 "{label} resolved to multiple items in vocabulary {}; use vocabulary_item_id",
9197 vocabulary_id.0
9198 ))),
9199 }
9200 }
9201
9202 async fn validate_vocabulary_item_reference_context(
9203 &self,
9204 label: &str,
9205 item: &ahri_tre_types::VocabularyItemRecord,
9206 reference: &VocabularyItemReference,
9207 ) -> Result<(), AppError> {
9208 if let Some(value) = reference.value
9209 && item.value != value
9210 {
9211 return Err(AppError::Validation(format!(
9212 "{label} {} has value {}, not {}",
9213 item.vocabulary_item_id.0, item.value, value
9214 )));
9215 }
9216 if let Some(code) = &reference.code
9217 && &item.code != code
9218 {
9219 return Err(AppError::Validation(format!(
9220 "{label} {} has code {}, not {}",
9221 item.vocabulary_item_id.0, item.code, code
9222 )));
9223 }
9224 if reference.vocabulary_id.is_some()
9225 || reference.vocabulary_name.is_some()
9226 || reference.domain_id.is_some()
9227 {
9228 let vocabulary_id = self.resolve_vocabulary_reference(label, reference).await?;
9229 if item.vocabulary_id != vocabulary_id {
9230 return Err(AppError::Validation(format!(
9231 "{label} {} belongs to vocabulary {}, not {}",
9232 item.vocabulary_item_id.0, item.vocabulary_id.0, vocabulary_id.0
9233 )));
9234 }
9235 }
9236 Ok(())
9237 }
9238
9239 async fn resolve_vocabulary_reference(
9240 &self,
9241 label: &str,
9242 reference: &VocabularyItemReference,
9243 ) -> Result<ahri_tre_types::VocabularyId, AppError> {
9244 match (
9245 reference.vocabulary_id,
9246 reference.domain_id,
9247 &reference.vocabulary_name,
9248 ) {
9249 (Some(vocabulary_id), domain_id, name) => {
9250 let vocabulary = self
9251 .vocabularies
9252 .get_vocabulary(vocabulary_id)
9253 .await
9254 .map_err(|error| core_error("lookup vocabulary", error))?
9255 .ok_or_else(|| {
9256 AppError::Validation(format!(
9257 "{label} vocabulary {} not found",
9258 vocabulary_id.0
9259 ))
9260 })?;
9261 if let Some(domain_id) = domain_id
9262 && vocabulary.domain_id != domain_id
9263 {
9264 return Err(AppError::Validation(format!(
9265 "{label} vocabulary {} belongs to domain {}, not {}",
9266 vocabulary_id.0, vocabulary.domain_id.0, domain_id.0
9267 )));
9268 }
9269 if let Some(name) = name
9270 && &vocabulary.name != name
9271 {
9272 return Err(AppError::Validation(format!(
9273 "{label} vocabulary {} is named {}, not {}",
9274 vocabulary_id.0,
9275 vocabulary.name.as_str(),
9276 name.as_str()
9277 )));
9278 }
9279 Ok(vocabulary_id)
9280 }
9281 (None, Some(domain_id), Some(name)) => self
9282 .vocabularies
9283 .list_vocabularies(domain_id)
9284 .await
9285 .map_err(|error| core_error("list vocabularies", error))?
9286 .into_iter()
9287 .find(|vocabulary| &vocabulary.name == name)
9288 .map(|vocabulary| vocabulary.vocabulary_id)
9289 .ok_or_else(|| {
9290 AppError::Validation(format!(
9291 "{label} vocabulary {} not found in domain {}",
9292 name.as_str(),
9293 domain_id.0
9294 ))
9295 }),
9296 (None, Some(_), None) => Err(AppError::Validation(format!(
9297 "{label} requires vocabulary_id or vocabulary_name"
9298 ))),
9299 (None, None, Some(_)) => Err(AppError::Validation(format!(
9300 "{label} vocabulary_name requires domain_id"
9301 ))),
9302 (None, None, None) => Err(AppError::Validation(format!(
9303 "{label} requires vocabulary_id or domain_id plus vocabulary_name"
9304 ))),
9305 }
9306 }
9307
9308 fn sql_analysis_variable_registrations(
9309 &self,
9310 domain_id: ahri_tre_types::DomainId,
9311 analysis: &SqlSourceAnalysis,
9312 ) -> Result<Vec<DatasetVariableRegistrationInput>, AppError> {
9313 let mut registrations = Vec::with_capacity(analysis.columns.len());
9314 for column in &analysis.columns {
9315 let variable_name = ahri_tre_types::NcName::parse(&column.name).map_err(|error| {
9316 AppError::Validation(format!(
9317 "SQL source column {} is not a valid variable name: {error}",
9318 column.name
9319 ))
9320 })?;
9321 let vocabulary = column
9322 .vocabulary
9323 .as_ref()
9324 .map(|vocabulary| {
9325 let name =
9326 ahri_tre_types::NcName::parse(&vocabulary.name).map_err(|error| {
9327 AppError::Validation(format!(
9328 "SQL source vocabulary {} is not a valid vocabulary name: {error}",
9329 vocabulary.name
9330 ))
9331 })?;
9332 Ok::<_, AppError>(ahri_tre_types::NewVocabularyRecord {
9333 domain_id,
9334 name,
9335 description: vocabulary.description.clone(),
9336 })
9337 })
9338 .transpose()?;
9339 let vocabulary_items = match &column.vocabulary {
9340 Some(inferred) => inferred
9341 .items
9342 .iter()
9343 .map(|item| ahri_tre_types::NewVocabularyItemRecord {
9344 value: item.value,
9345 code: item.code.clone(),
9346 description: item.description.clone(),
9347 })
9348 .collect(),
9349 None => Vec::new(),
9350 };
9351 registrations.push(DatasetVariableRegistrationInput::New(
9352 NewDatasetVariableRegistration {
9353 variable: ahri_tre_types::NewVariableRecord {
9354 domain_id,
9355 name: variable_name,
9356 value_type_id: column.value_type_id,
9357 value_format: None,
9358 vocabulary_id: None,
9359 key_role: ahri_tre_types::KeyRole::None,
9360 description: column.description.clone(),
9361 note: if analysis.warnings.is_empty() {
9362 None
9363 } else {
9364 Some(analysis.warnings.join("; "))
9365 },
9366 ontology_namespace: None,
9367 ontology_class: None,
9368 },
9369 row_role: Some(ahri_tre_types::KeyRole::None),
9370 vocabulary,
9371 vocabulary_items,
9372 source_column_name: None,
9373 },
9374 ));
9375 }
9376 Ok(registrations)
9377 }
9378
9379 fn explicit_registration_metadata_warnings(
9380 source_format: DatasetFileFormat,
9381 columns: &[ImportedColumn],
9382 explicit_registrations_supplied: bool,
9383 ) -> Vec<String> {
9384 if !explicit_registrations_supplied || columns.is_empty() {
9385 return Vec::new();
9386 }
9387
9388 if automatic_fallback_variable_format(source_format) {
9389 let format_name = dataset_file_format_name(source_format);
9390 vec![format!(
9391 "explicit variable registrations supplied; ignored automatic/discovered {format_name} metadata for {} imported columns",
9392 columns.len()
9393 )]
9394 } else {
9395 Vec::new()
9396 }
9397 }
9398
9399 async fn automatic_metadata_conflict_warnings(
9400 &self,
9401 study_id: ahri_tre_types::StudyId,
9402 metadata_domain_id: Option<ahri_tre_types::DomainId>,
9403 variable_registrations: &[DatasetVariableRegistrationInput],
9404 strict_metadata: bool,
9405 force_metadata_updates: bool,
9406 ) -> Result<Vec<String>, AppError> {
9407 if variable_registrations.is_empty() {
9408 return Ok(Vec::new());
9409 }
9410 let domain_id = if let Some(domain_id) = metadata_domain_id {
9411 self.ensure_study_domain(study_id, domain_id, "automatic metadata")
9412 .await?;
9413 domain_id
9414 } else {
9415 let domains = self
9416 .study_domains
9417 .list_domains_for_study(study_id)
9418 .await
9419 .map_err(|error| {
9420 core_error("list study domains for automatic metadata preflight", error)
9421 })?;
9422 let Some(domain) = domains.as_slice().first() else {
9423 return Ok(Vec::new());
9424 };
9425 domain.domain_id
9426 };
9427 let variables = self
9428 .variables
9429 .list_domain_variables(domain_id)
9430 .await
9431 .map_err(|error| {
9432 core_error(
9433 "list domain variables for automatic metadata preflight",
9434 error,
9435 )
9436 })?;
9437 let vocabularies = self
9438 .vocabularies
9439 .list_vocabularies(domain_id)
9440 .await
9441 .map_err(|error| {
9442 core_error(
9443 "list domain vocabularies for automatic metadata preflight",
9444 error,
9445 )
9446 })?;
9447 let mut warnings = Vec::new();
9448 for registration in variable_registrations {
9449 let DatasetVariableRegistrationInput::New(registration) = registration else {
9450 continue;
9451 };
9452 if let Some(existing_variable) = variables
9453 .iter()
9454 .find(|candidate| candidate.name == registration.variable.name)
9455 {
9456 if force_metadata_updates
9457 && existing_variable.value_type_id != registration.variable.value_type_id
9458 {
9459 warnings.push(format!(
9460 "force_metadata_updates enabled: automatic metadata will update {} value_type_id from {} to {}",
9461 registration.variable.name.as_str(),
9462 existing_variable.value_type_id.0,
9463 registration.variable.value_type_id.0
9464 ));
9465 }
9466 if force_metadata_updates
9467 && existing_variable.vocabulary_id.is_none()
9468 && registration.vocabulary.is_some()
9469 {
9470 warnings.push(format!(
9471 "force_metadata_updates enabled: automatic metadata will attach governed vocabulary to {}",
9472 registration.variable.name.as_str()
9473 ));
9474 }
9475 compare_optional_enrichment(
9476 &mut warnings,
9477 strict_metadata,
9478 "automatic metadata conflict",
9479 registration.variable.name.as_str(),
9480 "description",
9481 existing_variable.description.as_deref(),
9482 registration.variable.description.as_deref(),
9483 )?;
9484 compare_optional_enrichment(
9485 &mut warnings,
9486 strict_metadata,
9487 "automatic metadata conflict",
9488 registration.variable.name.as_str(),
9489 "note",
9490 existing_variable.note.as_deref(),
9491 registration.variable.note.as_deref(),
9492 )?;
9493 compare_optional_enrichment(
9494 &mut warnings,
9495 strict_metadata,
9496 "automatic metadata conflict",
9497 registration.variable.name.as_str(),
9498 "ontology namespace",
9499 existing_variable.ontology_namespace.as_deref(),
9500 registration.variable.ontology_namespace.as_deref(),
9501 )?;
9502 compare_optional_enrichment(
9503 &mut warnings,
9504 strict_metadata,
9505 "automatic metadata conflict",
9506 registration.variable.name.as_str(),
9507 "ontology class",
9508 existing_variable.ontology_class.as_deref(),
9509 registration.variable.ontology_class.as_deref(),
9510 )?;
9511 }
9512 if let Some(proposed_vocabulary) = ®istration.vocabulary
9513 && let Some(existing_vocabulary) = vocabularies
9514 .iter()
9515 .find(|candidate| candidate.name == proposed_vocabulary.name)
9516 {
9517 compare_optional_enrichment(
9518 &mut warnings,
9519 strict_metadata,
9520 "automatic metadata conflict",
9521 proposed_vocabulary.name.as_str(),
9522 "vocabulary description",
9523 existing_vocabulary.description.as_deref(),
9524 proposed_vocabulary.description.as_deref(),
9525 )?;
9526 }
9527 }
9528 Ok(warnings)
9529 }
9530
9531 async fn automatic_column_shape_variable_registrations(
9532 &self,
9533 study_id: ahri_tre_types::StudyId,
9534 metadata_domain_id: Option<ahri_tre_types::DomainId>,
9535 format: DatasetFileFormat,
9536 columns: &[ImportedColumn],
9537 field_metadata_proposals: Option<&[FieldMetadataProposal]>,
9538 profiled_vocabularies: &BTreeMap<String, Vec<String>>,
9539 ) -> Result<(Vec<DatasetVariableRegistrationInput>, Vec<String>), AppError> {
9540 let format_name = dataset_file_format_name(format);
9541 let domain_id = self
9542 .resolve_automatic_metadata_domain(study_id, metadata_domain_id, format)
9543 .await?;
9544
9545 let mut registrations = Vec::with_capacity(columns.len());
9546 let mut warnings = Vec::with_capacity(columns.len());
9547 let mut used_names = std::collections::BTreeSet::new();
9548 let proposals_by_field = field_metadata_proposals
9549 .unwrap_or_default()
9550 .iter()
9551 .map(|proposal| (proposal.field_name.as_str(), proposal))
9552 .collect::<BTreeMap<_, _>>();
9553 for column in columns {
9554 let proposal = proposals_by_field.get(column.name.as_str()).copied();
9555 let profiled_vocabulary = if proposal.is_some_and(field_metadata_proposal_is_rich) {
9556 None
9557 } else {
9558 profiled_vocabularies.get(column.name.as_str())
9559 };
9560 let variable_name = proposal
9561 .and_then(|proposal| proposal.variable_name.as_deref())
9562 .unwrap_or(column.name.as_str());
9563 let name = sanitized_automatic_variable_name(variable_name, &mut used_names)?;
9564 let has_vocabulary = proposal.is_some_and(|proposal| {
9565 !proposal.vocabulary_items.is_empty()
9566 || proposal
9567 .value_type
9568 .as_deref()
9569 .is_some_and(is_enumeration_value_type)
9570 || proposal.value_type_id == Some(7)
9571 }) || profiled_vocabulary.is_some();
9572 let value_type_id = if has_vocabulary {
9573 ahri_tre_types::ValueTypeId(7)
9574 } else {
9575 proposal
9576 .map(metadata_value_type_id)
9577 .transpose()?
9578 .flatten()
9579 .unwrap_or_else(|| value_type_id_for_duckdb_type(&column.duckdb_type))
9580 };
9581 let vocabulary = if proposal
9582 .is_some_and(|proposal| !proposal.vocabulary_items.is_empty())
9583 || profiled_vocabulary.is_some()
9584 {
9585 let vocabulary_name = proposal
9586 .and_then(|proposal| proposal.vocabulary_name.clone())
9587 .unwrap_or_else(|| vocabulary_name_for_variable(name.as_str()));
9588 Some(ahri_tre_types::NewVocabularyRecord {
9589 domain_id,
9590 name: sanitized_automatic_variable_name(
9591 &vocabulary_name,
9592 &mut std::collections::BTreeSet::new(),
9593 )?,
9594 description: proposal
9595 .and_then(|proposal| proposal.vocabulary_description.clone()),
9596 })
9597 } else {
9598 None
9599 };
9600 let vocabulary_items = if let Some(proposal) = proposal
9601 && !proposal.vocabulary_items.is_empty()
9602 {
9603 let mut items = Vec::with_capacity(proposal.vocabulary_items.len());
9604 for (index, item) in proposal.vocabulary_items.iter().enumerate() {
9605 let value = i32::try_from(index + 1).map_err(|_| {
9606 AppError::Validation(format!(
9607 "too many vocabulary items for automatic {format_name} column {}",
9608 column.name
9609 ))
9610 })?;
9611 items.push(ahri_tre_types::NewVocabularyItemRecord {
9612 value,
9613 code: item.code.clone(),
9614 description: item.description.clone(),
9615 });
9616 }
9617 items
9618 } else if let Some(values) = profiled_vocabulary {
9619 values
9620 .iter()
9621 .enumerate()
9622 .map(|(index, value)| {
9623 Ok(ahri_tre_types::NewVocabularyItemRecord {
9624 value: i32::try_from(index + 1).map_err(|_| {
9625 AppError::Validation(format!(
9626 "too many vocabulary items for automatic {format_name} column {}",
9627 column.name
9628 ))
9629 })?,
9630 code: value.clone(),
9631 description: Some(value.clone()),
9632 })
9633 })
9634 .collect::<Result<Vec<_>, AppError>>()?
9635 } else {
9636 Vec::new()
9637 };
9638 if let Some(proposal) = proposal {
9639 warnings.extend(proposal.warnings.iter().map(|warning| {
9640 format!(
9641 "{} field {} metadata warning: {warning}",
9642 format_name, column.name
9643 )
9644 }));
9645 }
9646 let rich_metadata_note = if profiled_vocabulary.is_some() {
9647 "bounded distinct-value profiling inferred governed vocabulary"
9648 } else if proposal.is_some_and(|proposal| {
9649 proposal.description.is_some() || !proposal.vocabulary_items.is_empty()
9650 }) {
9651 match format {
9652 DatasetFileFormat::ArrowIpc => "rich Arrow IPC field metadata applied",
9653 DatasetFileFormat::Parquet => "rich Parquet field metadata applied",
9654 _ => "rich field metadata applied",
9655 }
9656 } else {
9657 "no rich metadata extracted"
9658 };
9659 warnings.push(format!(
9660 "automatically registered {format_name} column {} as {} from DuckDB imported column shape type {} using TRE variable {}; {rich_metadata_note}",
9661 column.name,
9662 value_type_name(value_type_id)?,
9663 column.duckdb_type,
9664 name.as_str()
9665 ));
9666 registrations.push(DatasetVariableRegistrationInput::New(
9667 NewDatasetVariableRegistration {
9668 variable: ahri_tre_types::NewVariableRecord {
9669 domain_id,
9670 name,
9671 value_type_id,
9672 value_format: None,
9673 vocabulary_id: None,
9674 key_role: ahri_tre_types::KeyRole::None,
9675 description: proposal.and_then(|proposal| proposal.description.clone()),
9676 note: None,
9677 ontology_namespace: None,
9678 ontology_class: None,
9679 },
9680 row_role: Some(ahri_tre_types::KeyRole::None),
9681 vocabulary,
9682 vocabulary_items,
9683 source_column_name: Some(column.name.clone()),
9684 },
9685 ));
9686 }
9687 Ok((registrations, warnings))
9688 }
9689
9690 fn field_metadata_proposals(
9691 format: DatasetFileFormat,
9692 path: &Path,
9693 ) -> Result<(Vec<FieldMetadataProposal>, Vec<String>), AppError> {
9694 if format == DatasetFileFormat::Json {
9695 let extraction = ahri_tre_tabular::extract_json_file_metadata_proposals(path)
9696 .map_err(|e| infrastructure_error("read JSON field metadata", e))?;
9697 return Ok((extraction.proposals, extraction.warnings));
9698 }
9699 let schema = match format {
9700 DatasetFileFormat::ArrowIpc => read_ipc_schema(path)
9701 .map_err(|e| infrastructure_error("read Arrow field metadata", e))?,
9702 DatasetFileFormat::Parquet => read_parquet_schema(path)
9703 .map_err(|e| infrastructure_error("read Parquet field metadata", e))?,
9704 _ => return Ok((Vec::new(), Vec::new())),
9705 };
9706 Ok((
9707 ahri_tre_tabular::extract_field_metadata_proposals(schema.as_ref()),
9708 Vec::new(),
9709 ))
9710 }
9711
9712 async fn resolve_redcap_study(
9713 &self,
9714 project_info: &RedcapProjectInfo,
9715 context: RedcapStudyResolveContext<'_>,
9716 ) -> Result<ahri_tre_types::StudyRecord, AppError> {
9717 if let Some(study_id) = context.study_id {
9718 return self
9719 .studies
9720 .get_study_by_id(study_id)
9721 .await
9722 .map_err(|error| core_error("lookup REDCap study", error))?
9723 .ok_or_else(|| {
9724 AppError::Validation(format!("REDCap study not found: {}", study_id.0))
9725 });
9726 }
9727 let study_name = context
9728 .study_name
9729 .cloned()
9730 .map(Ok)
9731 .unwrap_or_else(|| ahri_tre_types::NcName::parse(project_info.default_study_name()))
9732 .map_err(|error| {
9733 AppError::Validation(format!("REDCap study name is not a valid NCName: {error}"))
9734 })?;
9735 if let Some(existing) = self
9736 .studies
9737 .get_study_by_name(study_name.as_str())
9738 .await
9739 .map_err(|error| core_error("lookup REDCap study by generated name", error))?
9740 {
9741 return Ok(existing);
9742 }
9743 let registration = self
9744 .register_new_study(RegisterNewStudyRequest {
9745 study: ahri_tre_types::NewStudyRecord {
9746 name: study_name,
9747 description: Some(project_info.project_title.clone()),
9748 documentation: Some(format!(
9749 "Created from REDCap project {} ({})",
9750 project_info.project_id, project_info.project_title
9751 )),
9752 agent_instructions: context.agent_instructions.cloned(),
9753 external_id: Some(project_info.project_id.clone()),
9754 study_type_id: None,
9755 date_created: None,
9756 created_by: context.ingest_transformation.created_by.clone(),
9757 },
9758 domain_ids: vec![context.domain_id],
9759 })
9760 .await?;
9761 Ok(registration.study)
9762 }
9763
9764 async fn ensure_redcap_domain(
9765 &self,
9766 project_info: &RedcapProjectInfo,
9767 domain_id: Option<ahri_tre_types::DomainId>,
9768 domain_name: Option<&ahri_tre_types::NcName>,
9769 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
9770 if let Some(domain_id) = domain_id
9771 && let Some(domain) = self
9772 .domains
9773 .get_domain_by_id(domain_id)
9774 .await
9775 .map_err(|error| core_error("lookup REDCap domain by id", error))?
9776 {
9777 return Ok(domain);
9778 }
9779
9780 let domain_name = domain_name
9781 .cloned()
9782 .map(Ok)
9783 .unwrap_or_else(|| ahri_tre_types::NcName::parse(redcap_domain_name(project_info)))
9784 .map_err(|error| {
9785 AppError::Validation(format!("REDCap domain name is not a valid NCName: {error}"))
9786 })?;
9787 if let Some(domain) = self
9788 .domains
9789 .get_domain_by_name(domain_name.as_str())
9790 .await
9791 .map_err(|error| core_error("lookup REDCap domain by name", error))?
9792 {
9793 if let Some(domain_id) = domain_id
9794 && domain.domain_id != domain_id
9795 {
9796 return Err(AppError::Validation(format!(
9797 "REDCap domain {} exists with domain id {}, expected {}",
9798 domain.name.as_str(),
9799 domain.domain_id.0,
9800 domain_id.0
9801 )));
9802 }
9803 return Ok(domain);
9804 }
9805
9806 self.register_or_create_redcap_domain(
9807 domain_id,
9808 domain_name,
9809 Some(format!(
9810 "REDCap project {} metadata domain for {}",
9811 project_info.project_id, project_info.project_title
9812 )),
9813 Some(format!("redcap:project:{}", project_info.project_id)),
9814 )
9815 .await
9816 }
9817
9818 async fn register_or_create_redcap_domain(
9819 &self,
9820 domain_id: Option<ahri_tre_types::DomainId>,
9821 name: ahri_tre_types::NcName,
9822 description: Option<String>,
9823 uri: Option<String>,
9824 ) -> Result<ahri_tre_types::DomainRecord, AppError> {
9825 match domain_id {
9826 Some(domain_id) => {
9827 let domain = self
9828 .register_domain(RegisterDomainRequest {
9829 domain: ahri_tre_types::DomainRecord {
9830 domain_id,
9831 name,
9832 description,
9833 uri,
9834 },
9835 })
9836 .await?;
9837 if domain.domain_id != domain_id {
9838 return Err(AppError::Validation(format!(
9839 "REDCap domain name {} already exists with domain id {}, expected {}",
9840 domain.name.as_str(),
9841 domain.domain_id.0,
9842 domain_id.0
9843 )));
9844 }
9845 Ok(domain)
9846 }
9847 None => {
9848 self.register_new_domain(RegisterNewDomainRequest {
9849 domain: ahri_tre_types::NewDomainRecord {
9850 name,
9851 description,
9852 uri,
9853 },
9854 })
9855 .await
9856 }
9857 }
9858 }
9859
9860 async fn ensure_study_domain(
9861 &self,
9862 study_id: ahri_tre_types::StudyId,
9863 domain_id: ahri_tre_types::DomainId,
9864 context: &str,
9865 ) -> Result<(), AppError> {
9866 let domains = self
9867 .study_domains
9868 .list_domains_for_study(study_id)
9869 .await
9870 .map_err(|error| core_error("list study domains", error))?;
9871 if domains.iter().any(|domain| domain.domain_id == domain_id) {
9872 Ok(())
9873 } else {
9874 Err(AppError::Validation(format!(
9875 "{context} domain {} is not linked to study {}",
9876 domain_id.0, study_id.0
9877 )))
9878 }
9879 }
9880
9881 async fn stage_redcap_artifact(
9882 &self,
9883 context: RedcapArtifactStageContext<'_>,
9884 artifact: RedcapArtifactInput,
9885 ) -> Result<StagedRedcapArtifact, AppError> {
9886 if !artifact.path.is_file() {
9887 return Err(AppError::Validation(format!(
9888 "REDCap artifact file not found: {}",
9889 artifact.path.display()
9890 )));
9891 }
9892 let asset_name =
9893 ahri_tre_types::NcName::parse(format!("{}_{}", context.asset_prefix, artifact.suffix))
9894 .map_err(|error| {
9895 AppError::Validation(format!("REDCap artifact asset name is invalid: {error}"))
9896 })?;
9897 let existing_asset = self
9898 .assets
9899 .get_asset_by_name(context.study_id, asset_name.as_str())
9900 .await
9901 .map_err(|error| core_error("lookup REDCap artifact asset", error))?;
9902 let (asset, asset_persisted) = match existing_asset {
9903 Some(asset) => {
9904 if asset.asset_type != ahri_tre_types::AssetType::File {
9905 return Err(AppError::Validation(format!(
9906 "REDCap artifact asset {} is not a file asset",
9907 asset.asset_id.0
9908 )));
9909 }
9910 (asset, true)
9911 }
9912 None => {
9913 let new_asset = ahri_tre_types::NewAssetRecord {
9914 study_id: context.study_id,
9915 name: asset_name,
9916 description: Some(artifact.description.clone()),
9917 agent_instructions: context.agent_instructions.cloned(),
9918 asset_type: ahri_tre_types::AssetType::File,
9919 date_created: None,
9920 created_by: context.created_by.cloned(),
9921 };
9922 let asset = self
9923 .assets
9924 .create_asset(&new_asset)
9925 .await
9926 .map_err(|error| core_error("create REDCap artifact asset", error))?;
9927 (asset, true)
9928 }
9929 };
9930 let existing_versions = self
9931 .assets
9932 .list_asset_versions(asset.asset_id)
9933 .await
9934 .map_err(|error| core_error("list REDCap artifact versions", error))?;
9935 let (major, minor, patch) =
9936 ahri_tre_core::next_asset_version_coordinates(&existing_versions)
9937 .map_err(|error| core_error("choose REDCap artifact version", error))?;
9938 let version = ahri_tre_types::NewAssetVersionRecord {
9939 classification: context.classification.clone(),
9940 asset_id: asset.asset_id,
9941 major,
9942 minor,
9943 patch,
9944 version_label: None,
9945 version_note: context.version_note.cloned(),
9946 is_latest: None,
9947 doi: None,
9948 created_at: None,
9949 created_by: context.created_by.cloned(),
9950 };
9951 ahri_tre_core::validate_new_asset_version_input(&existing_versions, &version)
9952 .map_err(|error| core_error("validate REDCap artifact version", error))?;
9953 let version = ahri_tre_core::prepare_new_asset_version_for_create(version)
9954 .map_err(|error| core_error("prepare REDCap artifact version", error))?;
9955 let version = self
9956 .assets
9957 .create_asset_version(&version)
9958 .await
9959 .map_err(|error| core_error("create REDCap artifact version", error))?;
9960 let mut source = RedcapArtifactReader {
9961 file: std::fs::File::open(&artifact.path)
9962 .map_err(|error| infrastructure_error("open verified REDCap role", error))?,
9963 limits: context.limits,
9964 };
9965 let staged = DuckLakeAdapter::new(context.lake_root)
9966 .stage_datafile_stream(
9967 &StageDataFileStreamRequest {
9968 source_file_name: artifact
9969 .path
9970 .file_name()
9971 .and_then(|name| name.to_str())
9972 .ok_or_else(|| AppError::Validation("Invalid REDCap role filename".into()))?
9973 .to_owned(),
9974 asset_id: asset.asset_id,
9975 version_id: version.version_id,
9976 study_id: context.study_id,
9977 asset_name: asset.name.clone(),
9978 major,
9979 minor,
9980 patch,
9981 edam_format: artifact.edam_format.to_string(),
9982 compression: if context.compress {
9983 DataFileCompression::Zstd
9984 } else {
9985 DataFileCompression::None
9986 },
9987 encryption: if context.encrypt {
9988 DataFileEncryption::Aes256Cbc
9989 } else {
9990 DataFileEncryption::None
9991 },
9992 },
9993 &mut source,
9994 )
9995 .map_err(|error| infrastructure_error("stage REDCap artifact", error))?;
9996 if !asset_persisted {
9997 self.assets
9998 .save_asset(&asset)
9999 .await
10000 .map_err(|error| core_error("save REDCap artifact asset", error))?;
10001 }
10002 Self::bind_staged_datafile(
10003 context.operator,
10004 context.lake_root.to_owned(),
10005 asset.study_id,
10006 asset.asset_id,
10007 staged.datafile.clone(),
10008 )
10009 .await?;
10010 let datafile = self
10011 .assets
10012 .save_datafile(&staged.datafile)
10013 .await
10014 .map_err(|error| core_error("save REDCap artifact datafile", error))?;
10015 Ok(StagedRedcapArtifact {
10016 suffix: artifact.suffix.to_string(),
10017 asset,
10018 version,
10019 datafile,
10020 source_size_bytes: staged.source_size_bytes,
10021 stored_size_bytes: staged.staged_size_bytes,
10022 })
10023 }
10024
10025 #[allow(clippy::too_many_arguments)]
10026 async fn materialize_redcap_forms(
10027 &self,
10028 session: &mut DataStoreSession,
10029 lake: &DuckLakeAdapter,
10030 eav_datafile: &ahri_tre_types::DataFileRecord,
10031 study: &ahri_tre_types::StudyRecord,
10032 domain_id: ahri_tre_types::DomainId,
10033 project_info: &RedcapProjectInfo,
10034 forms: &[RedcapForm],
10035 artifact_versions: &[ahri_tre_types::VersionId],
10036 options: RedcapMaterializeOptions<'_>,
10037 ) -> Result<RedcapFormsResult, AppError> {
10038 use ahri_tre_protocol::ingest::{RedcapFormOutcome, RedcapFormOutcomeKind as Outcome};
10039 let mut outcomes = Vec::new();
10040 let mut datasets = Vec::new();
10041 let mut warnings = Vec::new();
10042 for form in forms {
10045 for field in &form.fields {
10046 warnings.extend(redcap_variable_registration(domain_id, field)?.1);
10047 }
10048 }
10049 for form in forms {
10050 let dataset_name = redcap_dataset_name(project_info, form, options.dataset_prefix)?;
10051 let mut outcome = RedcapFormOutcome {
10052 dataset: dataset_name.as_str().into(),
10053 outcome: Outcome::NotMaterialized,
10054 };
10055 if options
10056 .limits
10057 .is_some_and(|limits| limits.check_deadline().is_err())
10058 {
10059 warnings.push("REDCap remaining forms interrupted before admission".into());
10060 outcomes.push(outcome);
10061 continue;
10062 }
10063 let metadata_deadline = options
10064 .limits
10065 .and_then(|limits| {
10066 session
10067 .dataset_admission_repository()
10068 .map(|repo| repo.upload_deadline(limits.deadline, limits.cancelled.clone()))
10069 })
10070 .transpose()
10071 .map_err(|error| core_error("bound REDCap metadata", error))?;
10072 let monitor = options.limits.map(|limits| {
10073 upload::UploadMonitor::new(session, limits.deadline, limits.cancelled.clone())
10074 });
10075 let attempt = self
10076 .prepare_dataset_attempt(
10077 session,
10078 PrepareDatasetVersionRequest {
10079 output_intent: None,
10080 inherit_risk: false,
10081 classification: ahri_tre_types::IngestClassification::derived_high(),
10082 study_id: study.study_id,
10083 dataset_asset_id: None,
10084 dataset_name: dataset_name.clone(),
10085 dataset_version_id: None,
10086 description: Some(format!(
10087 "REDCap project {} form {} dataset",
10088 project_info.project_id, form.name
10089 )),
10090 agent_instructions: options.agent_instructions.cloned(),
10091 version_note: options.version_note.cloned(),
10092 created_by: options.transform_transformation.created_by.clone(),
10093 },
10094 )
10095 .await;
10096 let mut attempt = match attempt {
10097 Ok(attempt) => attempt,
10098 Err(error) => {
10099 if !matches!(error, AppError::Conflict(_)) {
10100 outcome.outcome = Outcome::InspectionRequired;
10101 }
10102 warnings.push(format!(
10103 "REDCap form {} could not prepare output; inspect ownership before retry",
10104 form.name
10105 ));
10106 outcomes.push(outcome);
10107 continue;
10108 }
10109 };
10110 let prepared = attempt.prepared.clone();
10111 let result = async {
10112 let source = lake
10113 .restore_dataset_source(
10114 &attempt.scratch,
10115 eav_datafile,
10116 DatasetFileFormat::Csv,
10117 )
10118 .map_err(|error| infrastructure_error("restore REDCap Dataset source", error))?;
10119 let preparation = async {
10120 let eav_path = source.path();
10121 let form_field_names = form
10122 .fields
10123 .iter()
10124 .map(|field| field.field_name.as_str())
10125 .collect::<Vec<_>>();
10126 let present_fields =
10127 redcap_eav_present_fields(session.lake_connection(), eav_path, &form_field_names)?;
10128 let selected_fields = form
10129 .fields
10130 .iter()
10131 .filter(|field| present_fields.contains(field.field_name.as_str()))
10132 .cloned()
10133 .collect::<Vec<_>>();
10134 let missing_fields = form
10135 .fields
10136 .iter()
10137 .filter(|field| !present_fields.contains(field.field_name.as_str()))
10138 .map(|field| field.field_name.clone())
10139 .collect::<Vec<_>>();
10140 if !missing_fields.is_empty() {
10141 warnings.push(format!(
10142 "REDCap form {} metadata fields were absent from EAV export and were not linked to the dataset: {}",
10143 form.name,
10144 missing_fields.join(", ")
10145 ));
10146 }
10147 if selected_fields.is_empty() {
10148 warnings.push(format!(
10149 "REDCap form {} had no fields present in the EAV export; no dataset was created",
10150 form.name
10151 ));
10152 return Ok(None);
10153 }
10154 let synthetic_columns = redcap_synthetic_columns(
10155 session.lake_connection(),
10156 eav_path,
10157 dataset_name.as_str(),
10158 )?;
10159
10160 let lake_fields = selected_fields
10161 .iter()
10162 .map(redcap_lake_field)
10163 .collect::<Vec<_>>();
10164 let loaded = lake
10165 .load_redcap_form_dataset_reserved(
10166 session.lake_connection(),
10167 &mut attempt.authority,
10168 &LoadRedcapFormDatasetRequest {
10169 study_id: study.study_id,
10170 dataset_name: prepared.asset.name.clone(),
10171 major: prepared.version.major,
10172 minor: prepared.version.minor,
10173 patch: prepared.version.patch,
10174 source_eav_path: eav_path.to_path_buf(),
10175 fields: lake_fields,
10176 synthetic_columns: synthetic_columns.clone(),
10177 },
10178 )
10179 .map_err(|error| infrastructure_error("load REDCap form dataset", error))?;
10180 Ok::<_, AppError>(Some((selected_fields, synthetic_columns, loaded)))
10181
10182 }.await;
10183 let source_cleanup = source.cleanup();
10184 let (selected_fields, synthetic_columns, loaded) = match preparation {
10185 Ok(Some(prepared)) => prepared,
10186 other => {
10187 source_cleanup.map_err(|error| {
10188 infrastructure_error("clean REDCap Dataset source", error)
10189 })?;
10190 match other {
10191 Ok(None) => return Ok(None),
10192 Err(error) => return Err(error),
10193 Ok(Some(_)) => unreachable!(),
10194 }
10195 }
10196 };
10197 if let Some(limits) = options.limits {
10198 limits.check_deadline().map_err(|_| AppError::Validation("REDCap deadline elapsed".into()))?;
10199 if loaded.row_count as u64 > limits.budgets.rows { return Err(AppError::Validation("REDCap form exceeds row budget".into())) }
10200 }
10201 let relation = loaded.relation.clone();
10202 let dataset_record = ahri_tre_types::DatasetRecord {
10203 dataset_id: prepared.version.version_id,
10204 };
10205 let transform = redcap_transform_record(options.transform_transformation, &form.name);
10206 let metadata_result = async {
10207 source_cleanup
10208 .map_err(|error| infrastructure_error("clean REDCap Dataset source", error))?;
10209 let (variable_registrations, mut registration_warnings) =
10210 redcap_dataset_variable_registrations(
10211 domain_id,
10212 &selected_fields,
10213 &synthetic_columns,
10214 )?;
10215 warnings.append(&mut registration_warnings);
10216 let (catalog, dataset, lineage, variables) = self
10217 .save_dataset_metadata_with_options(
10218 session,
10219 &mut attempt,
10220 dataset_record,
10221 &transform,
10222 artifact_versions,
10223 DatasetMetadataRegistration {
10224 redcap_dictionary: (!options.no_register_dictionary && datasets.is_empty()).then_some((domain_id, forms)),
10225 variable_registrations: if options.no_register_dictionary { Vec::new() } else { variable_registrations },
10226 loaded_column_names: Some(&loaded.column_names),
10227 force_metadata_updates: options.force_metadata_updates,
10228 },
10229 )
10230 .await?;
10231 Ok::<_, AppError>((catalog, dataset, lineage, variables))
10232 }
10233 .await;
10234 let (catalog, dataset, lineage, variables) = metadata_result?;
10235 Ok(Some(DatasetMaterialization {
10236 catalog,
10237 dataset,
10238 lineage,
10239 lake_relation: relation.qualified_name(),
10240 lake_schema: relation.schema_name,
10241 lake_table: relation.table_name,
10242 row_count: loaded.row_count,
10243 column_count: loaded.column_count,
10244 variables,
10245 metadata_warnings: Vec::new(),
10246 }))
10247 }.await;
10248 drop(monitor);
10249 drop(metadata_deadline);
10250 let finished = match result {
10251 Ok(None) => attempt.abandon(session).map(|()| None),
10252 other => attempt.finish(session, other),
10253 };
10254 match finished {
10255 Ok(Some(dataset)) => {
10256 outcome.outcome = Outcome::Complete;
10257 datasets.push(dataset);
10258 }
10259 Ok(None) => (),
10260 Err(_) => {
10261 outcome.outcome = Outcome::InspectionRequired;
10262 warnings.push(format!("REDCap form {} failed; inspect output before retry because admission or cleanup may require reconciliation", form.name));
10263 }
10264 }
10265 outcomes.push(outcome);
10266 }
10267 Ok((datasets, warnings, outcomes))
10268 }
10269
10270 pub fn new_noop() -> Self {
10271 Self::new(AppRepositories {
10272 registrations: Arc::new(NoopCatalogueRegistrationRepository),
10273 domains: Arc::new(NoopDomainRepository),
10274 studies: Arc::new(NoopStudyRepository),
10275 study_domains: Arc::new(NoopStudyDomainRepository),
10276 study_governance: Arc::new(NoopStudyGovernanceRepository),
10277 assets: Arc::new(NoopAssetRepository),
10278 variables: Arc::new(NoopVariableRepository),
10279 vocabularies: Arc::new(NoopVocabularyRepository),
10280 entities: Arc::new(NoopEntityRepository),
10281 transformations: Arc::new(NoopTransformationRepository),
10282 tags: Arc::new(NoopTagRepository),
10283 })
10284 }
10285}
10286
10287fn archive_delete_validation_error(
10288 code: impl Into<String>,
10289 message: impl Into<String>,
10290) -> ArchiveDeleteValidationError {
10291 ArchiveDeleteValidationError {
10292 code: code.into(),
10293 message: redact_archive_delete_message(&message.into()),
10294 }
10295}
10296
10297fn redact_archive_delete_message(message: &str) -> String {
10298 message
10299 .split_whitespace()
10300 .map(redact_archive_delete_token)
10301 .collect::<Vec<_>>()
10302 .join(" ")
10303}
10304
10305fn redact_archive_delete_token(token: &str) -> String {
10306 let lowered = token.to_ascii_lowercase();
10307 let trimmed = lowered.trim_matches(|ch: char| matches!(ch, ',' | ';' | ')' | '('));
10308 let sensitive_assignment = [
10309 "password=",
10310 "passwd=",
10311 "passfile=",
10312 "pgpassfile=",
10313 "token=",
10314 "access_token=",
10315 "refresh_token=",
10316 "client_secret=",
10317 "authorization=",
10318 ]
10319 .iter()
10320 .any(|needle| trimmed.contains(needle));
10321
10322 if sensitive_assignment || trimmed.starts_with("bearer") {
10323 return "<redacted_secret>".to_string();
10324 }
10325 if trimmed.starts_with("postgres://") || trimmed.starts_with("postgresql://") {
10326 return "<redacted_connection_uri>".to_string();
10327 }
10328 if (trimmed.starts_with("http://") || trimmed.starts_with("https://"))
10329 && (trimmed.contains('?')
10330 || trimmed.contains("signature")
10331 || trimmed.contains("token")
10332 || trimmed.contains("secret"))
10333 {
10334 return "<redacted_uri>".to_string();
10335 }
10336 if token.starts_with('/') || token.starts_with('~') || trimmed.contains("/.pgpass") {
10337 return "<redacted_path>".to_string();
10338 }
10339
10340 token.to_string()
10341}
10342
10343fn archive_delete_archive_preview(archive_required: bool) -> ArchiveDeleteStagePreview {
10344 if archive_required {
10345 ArchiveDeleteStagePreview::pending(
10346 true,
10347 "archive creation will be resolved by the executable archive/delete workflow",
10348 )
10349 } else {
10350 ArchiveDeleteStagePreview {
10351 required: false,
10352 disposition: ArchiveDeleteCleanupDisposition::NotRequired,
10353 retryable: false,
10354 summary: "forced no-archive delete does not require an archive stage".to_string(),
10355 }
10356 }
10357}
10358
10359fn study_archive_temp_dir(study_id: ahri_tre_types::StudyId) -> Result<PathBuf, AppError> {
10360 let path = env::temp_dir().join(format!(
10361 "ahri-tre-study-archive-{}-{}",
10362 study_id.0,
10363 Utc::now()
10364 .timestamp_nanos_opt()
10365 .unwrap_or_else(|| Utc::now().timestamp_millis())
10366 ));
10367 if path.exists() {
10368 return Err(AppError::Infrastructure(
10369 "study archive staging directory already exists".to_string(),
10370 ));
10371 }
10372 Ok(path)
10373}
10374
10375fn archive_stage_transformation(
10376 description: &str,
10377 actor: &str,
10378 transformation_type: ahri_tre_types::TransformationType,
10379) -> ahri_tre_types::NewTransformationRecord {
10380 ahri_tre_types::NewTransformationRecord {
10381 transformation_type,
10382 description: description.to_string(),
10383 repository_url: None,
10384 commit_hash: None,
10385 file_path: None,
10386 date_created: Some(Utc::now()),
10387 created_by: Some(actor.to_string()),
10388 }
10389}
10390
10391fn build_study_archive_bundle(
10392 manifest: &StudyArchiveManifest,
10393 study: &ahri_tre_types::StudyRecord,
10394 assets: &[ahri_tre_types::AssetRecord],
10395 asset_versions: &[ahri_tre_types::AssetVersionRecord],
10396 transformations: &[TransformationSummary],
10397 dataset_payloads: &[(PathBuf, ArchivedDatasetFileRecord)],
10398 datafile_payloads: &[(PathBuf, ArchivedDataFileRecord)],
10399) -> Result<Vec<u8>, AppError> {
10400 let mut compressed = Vec::new();
10401 {
10402 let encoder = zstd::stream::write::Encoder::new(&mut compressed, 0)
10403 .map_err(|error| infrastructure_error("create study archive zstd encoder", error))?;
10404 let mut archive = tar::Builder::new(encoder);
10405 append_json_to_archive(&mut archive, "manifest.json", manifest)?;
10406 append_json_to_archive(&mut archive, "metadata/study.json", study)?;
10407 append_json_to_archive(&mut archive, "metadata/domains.json", &manifest.domains)?;
10408 append_json_to_archive(&mut archive, "metadata/variables.json", &manifest.variables)?;
10409 append_json_to_archive(
10410 &mut archive,
10411 "metadata/vocabularies.json",
10412 &manifest.vocabularies,
10413 )?;
10414 append_json_to_archive(
10415 &mut archive,
10416 "metadata/vocabulary_items.json",
10417 &manifest.vocabulary_items,
10418 )?;
10419 append_json_to_archive(&mut archive, "metadata/entities.json", &manifest.entities)?;
10420 append_json_to_archive(
10421 &mut archive,
10422 "metadata/entity_relations.json",
10423 &manifest.entity_relations,
10424 )?;
10425 append_json_to_archive(&mut archive, "metadata/assets.json", assets)?;
10426 append_json_to_archive(&mut archive, "metadata/asset_versions.json", asset_versions)?;
10427 append_json_to_archive(
10428 &mut archive,
10429 "metadata/transformations.json",
10430 transformations,
10431 )?;
10432 for (path, record) in dataset_payloads {
10433 archive
10434 .append_path_with_name(path, &record.bundle_path)
10435 .map_err(|error| {
10436 infrastructure_error("append dataset payload to archive", error)
10437 })?;
10438 }
10439 for (path, record) in datafile_payloads {
10440 archive
10441 .append_path_with_name(path, &record.bundle_path)
10442 .map_err(|error| {
10443 infrastructure_error("append datafile payload to archive", error)
10444 })?;
10445 }
10446 let encoder = archive
10447 .into_inner()
10448 .map_err(|error| infrastructure_error("finish study archive tar stream", error))?;
10449 encoder
10450 .finish()
10451 .map_err(|error| infrastructure_error("finish study archive zstd stream", error))?;
10452 }
10453 Ok(compressed)
10454}
10455
10456fn append_json_to_archive<T: serde::Serialize + ?Sized>(
10457 archive: &mut tar::Builder<zstd::stream::write::Encoder<'_, &mut Vec<u8>>>,
10458 path: &str,
10459 value: &T,
10460) -> Result<(), AppError> {
10461 let bytes = serde_json::to_vec_pretty(value)
10462 .map_err(|error| infrastructure_error("serialize study archive JSON", error))?;
10463 let mut header = tar::Header::new_gnu();
10464 header.set_size(bytes.len() as u64);
10465 header.set_mode(0o644);
10466 header.set_cksum();
10467 archive
10468 .append_data(&mut header, path, Cursor::new(bytes))
10469 .map_err(|error| infrastructure_error("append JSON to study archive", error))
10470}
10471
10472fn resolve_ingest_edam_format(
10473 source_path: &Path,
10474 explicit: Option<&str>,
10475) -> Result<String, AppError> {
10476 if let Some(value) = explicit.map(str::trim).filter(|value| !value.is_empty()) {
10477 return Ok(match value.to_ascii_lowercase().as_str() {
10478 "csv" | "text/csv" | "edam:format_3752" => "EDAM:format_3752".to_string(),
10479 "json" | "application/json" | "edam:format_3464" => "EDAM:format_3464".to_string(),
10480 _ => value.to_string(),
10481 });
10482 }
10483
10484 let extension = source_path
10485 .extension()
10486 .and_then(|value| value.to_str())
10487 .map(str::to_ascii_lowercase)
10488 .ok_or_else(|| {
10489 AppError::Validation(format!(
10490 "ingest EDAM format is required for source without extension: {}",
10491 source_path.display()
10492 ))
10493 })?;
10494
10495 match extension.as_str() {
10496 "csv" => Ok("EDAM:format_3752".to_string()),
10497 "json" => Ok("EDAM:format_3464".to_string()),
10498 _ => Err(AppError::Validation(format!(
10499 "ingest EDAM format is required for unsupported source extension: {extension}"
10500 ))),
10501 }
10502}
10503
10504fn resolve_dataset_file_format(
10505 source_path: Option<&Path>,
10506 explicit: Option<&str>,
10507) -> Result<DatasetFileFormat, AppError> {
10508 if let Some(value) = explicit.map(str::trim).filter(|value| !value.is_empty()) {
10509 let normalized = value.to_ascii_lowercase();
10510 return match normalized.as_str() {
10511 "arrow" | "ipc" | "feather" | "application/vnd.apache.arrow.file" => {
10512 Ok(DatasetFileFormat::ArrowIpc)
10513 }
10514 "csv" | "text/csv" | "edam:format_3752" => Ok(DatasetFileFormat::Csv),
10515 "json" | "application/json" | "application/x-ndjson" | "edam:format_3464" => {
10516 Ok(DatasetFileFormat::Json)
10517 }
10518 "parquet" | "application/parquet" | "application/vnd.apache.parquet" => {
10519 Ok(DatasetFileFormat::Parquet)
10520 }
10521 "xlsx" | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => {
10522 Ok(DatasetFileFormat::Xlsx)
10523 }
10524 _ => Err(AppError::Validation(format!(
10525 "unsupported dataset input format: {value}"
10526 ))),
10527 };
10528 }
10529
10530 let source_path = source_path.ok_or_else(|| {
10531 AppError::Validation("dataset input format is required for managed datafiles".to_string())
10532 })?;
10533 let extension = source_path
10534 .extension()
10535 .and_then(|value| value.to_str())
10536 .map(str::to_ascii_lowercase)
10537 .ok_or_else(|| {
10538 AppError::Validation(format!(
10539 "dataset input format is required for source without extension: {}",
10540 source_path.display()
10541 ))
10542 })?;
10543
10544 match extension.as_str() {
10545 "arrow" | "ipc" | "feather" => Ok(DatasetFileFormat::ArrowIpc),
10546 "csv" => Ok(DatasetFileFormat::Csv),
10547 "json" | "ndjson" => Ok(DatasetFileFormat::Json),
10548 "parquet" => Ok(DatasetFileFormat::Parquet),
10549 "xlsx" => Ok(DatasetFileFormat::Xlsx),
10550 _ => Err(AppError::Validation(format!(
10551 "unsupported dataset source extension: {extension}"
10552 ))),
10553 }
10554}
10555
10556fn managed_datafile_input_format(
10557 source_format: &str,
10558 requested_format: Option<&str>,
10559) -> Result<String, AppError> {
10560 let declared = resolve_dataset_file_format(None, Some(source_format))?;
10561 let requested = resolve_dataset_file_format(None, requested_format.or(Some(source_format)))?;
10562 if declared != requested {
10563 return Err(AppError::Validation(
10564 "Datafile format and requested parser disagree".to_string(),
10565 ));
10566 }
10567 Ok(match declared {
10568 DatasetFileFormat::ArrowIpc => "arrow",
10569 DatasetFileFormat::Csv => "csv",
10570 DatasetFileFormat::Json => "json",
10571 DatasetFileFormat::Parquet => "parquet",
10572 DatasetFileFormat::Xlsx => "xlsx",
10573 }
10574 .to_string())
10575}
10576
10577fn automatic_fallback_variable_format(format: DatasetFileFormat) -> bool {
10578 matches!(
10579 format,
10580 DatasetFileFormat::Csv
10581 | DatasetFileFormat::ArrowIpc
10582 | DatasetFileFormat::Json
10583 | DatasetFileFormat::Parquet
10584 | DatasetFileFormat::Xlsx
10585 )
10586}
10587
10588fn metadata_bearing_dataset_file_format(format: DatasetFileFormat) -> bool {
10589 matches!(
10590 format,
10591 DatasetFileFormat::ArrowIpc | DatasetFileFormat::Json | DatasetFileFormat::Parquet
10592 )
10593}
10594
10595fn dataset_file_format_name(format: DatasetFileFormat) -> &'static str {
10596 match format {
10597 DatasetFileFormat::ArrowIpc => "Arrow IPC",
10598 DatasetFileFormat::Csv => "CSV",
10599 DatasetFileFormat::Json => "JSON",
10600 DatasetFileFormat::Parquet => "Parquet",
10601 DatasetFileFormat::Xlsx => "XLSX",
10602 }
10603}
10604
10605fn dataset_lake_export_format(format: DatasetExportFormat) -> ahri_tre_lake::DatasetExportFormat {
10606 match format {
10607 DatasetExportFormat::Csv => ahri_tre_lake::DatasetExportFormat::Csv,
10608 DatasetExportFormat::Ndjson => ahri_tre_lake::DatasetExportFormat::Ndjson,
10609 DatasetExportFormat::Parquet => ahri_tre_lake::DatasetExportFormat::Parquet,
10610 DatasetExportFormat::Arrow => ahri_tre_lake::DatasetExportFormat::ArrowIpc,
10611 }
10612}
10613
10614fn text_export_compressed(format: DatasetExportFormat, requested: bool) -> bool {
10615 matches!(
10616 format,
10617 DatasetExportFormat::Csv | DatasetExportFormat::Ndjson
10618 ) && requested
10619}
10620
10621fn export_destination_path(
10622 target_dir: &Path,
10623 asset_name: &str,
10624 version: &ahri_tre_types::AssetVersionRecord,
10625 extension: &str,
10626 compressed: bool,
10627) -> Result<PathBuf, AppError> {
10628 if target_dir.exists() && !target_dir.is_dir() {
10629 return Err(AppError::Validation(format!(
10630 "export target is not a directory: {}",
10631 target_dir.display()
10632 )));
10633 }
10634 fs::create_dir_all(target_dir)
10635 .map_err(|error| infrastructure_error("create export target directory", error))?;
10636
10637 let mut file_name = format!(
10638 "{}_{}_{}_{}.{}",
10639 asset_name.trim(),
10640 version.major,
10641 version.minor,
10642 version.patch,
10643 extension.trim_start_matches('.')
10644 );
10645 if compressed {
10646 file_name.push_str(".zst");
10647 }
10648 Ok(target_dir.join(file_name))
10649}
10650
10651fn ensure_export_destination(path: &Path, overwrite: bool) -> Result<(), AppError> {
10652 if path.exists() {
10653 if path.is_dir() {
10654 return Err(AppError::Validation(format!(
10655 "export target path is a directory: {}",
10656 path.display()
10657 )));
10658 }
10659 if !overwrite {
10660 return Err(AppError::Validation(format!(
10661 "export target already exists: {}",
10662 path.display()
10663 )));
10664 }
10665 fs::remove_file(path)
10666 .map_err(|error| infrastructure_error("remove existing export target", error))?;
10667 }
10668 Ok(())
10669}
10670
10671fn datafile_logical_extension(datafile: &ahri_tre_types::DataFileRecord) -> String {
10672 if let Some(extension) = datafile_storage_logical_extension(&datafile.storage_uri) {
10673 return extension;
10674 }
10675 edam_format_extension(&datafile.edam_format)
10676 .unwrap_or("bin")
10677 .to_string()
10678}
10679
10680fn resolve_requested_asset_version(
10681 catalog: &AssetVersionCatalog,
10682 selector: Option<&str>,
10683) -> Result<ahri_tre_types::AssetVersionRecord, AppError> {
10684 let selector = selector
10685 .map(str::trim)
10686 .filter(|value| !value.is_empty())
10687 .unwrap_or("latest");
10688 if selector.eq_ignore_ascii_case("latest") {
10689 return catalog.latest_version.clone().ok_or_else(|| {
10690 AppError::Validation(format!(
10691 "asset {} has no versions",
10692 catalog.asset.name.as_str()
10693 ))
10694 });
10695 }
10696
10697 let Some((major, minor, patch)) = parse_semver_selector(selector) else {
10698 return Err(AppError::Validation(format!(
10699 "version must be latest or semver x.y.z: {selector}"
10700 )));
10701 };
10702 catalog
10703 .versions
10704 .iter()
10705 .find(|version| version.major == major && version.minor == minor && version.patch == patch)
10706 .cloned()
10707 .ok_or_else(|| {
10708 AppError::Validation(format!(
10709 "asset version not found: {} {selector}",
10710 catalog.asset.name.as_str()
10711 ))
10712 })
10713}
10714
10715fn asset_version_label_text(version: &ahri_tre_types::AssetVersionRecord) -> String {
10716 version.version_label.clone().unwrap_or_else(|| {
10717 ahri_tre_core::asset_version_label(version.major, version.minor, version.patch)
10718 })
10719}
10720
10721fn asset_type_label(asset_type: ahri_tre_types::AssetType) -> &'static str {
10722 match asset_type {
10723 ahri_tre_types::AssetType::Dataset => "dataset",
10724 ahri_tre_types::AssetType::File => "file",
10725 }
10726}
10727
10728fn parse_semver_selector(selector: &str) -> Option<(i32, i32, i32)> {
10729 let selector = selector.strip_prefix('v').unwrap_or(selector);
10730 let mut parts = selector.split('.');
10731 let major = parse_nonnegative_i32(parts.next()?)?;
10732 let minor = parse_nonnegative_i32(parts.next()?)?;
10733 let patch = parse_nonnegative_i32(parts.next()?)?;
10734 if parts.next().is_some() {
10735 return None;
10736 }
10737 Some((major, minor, patch))
10738}
10739
10740fn parse_nonnegative_i32(value: &str) -> Option<i32> {
10741 if value.is_empty() || !value.chars().all(|character| character.is_ascii_digit()) {
10742 return None;
10743 }
10744 value.parse().ok()
10745}
10746
10747fn datafile_storage_logical_extension(storage_uri: &str) -> Option<String> {
10748 let file_name = storage_uri
10749 .rsplit(['/', '\\'])
10750 .next()
10751 .map(str::trim)
10752 .filter(|value| !value.is_empty())?;
10753 let path = Path::new(file_name);
10754 let extension = path.extension()?.to_str()?.trim().to_ascii_lowercase();
10755 if extension == "zst" {
10756 let stem = path.file_stem()?.to_str()?;
10757 return Path::new(stem)
10758 .extension()
10759 .and_then(|value| value.to_str())
10760 .map(|value| value.to_ascii_lowercase());
10761 }
10762 Some(extension)
10763}
10764
10765fn edam_format_extension(format: &str) -> Option<&'static str> {
10766 match format.trim().to_ascii_lowercase().as_str() {
10767 "edam:format_3752" | "text/csv" | "csv" => Some("csv"),
10768 "edam:format_3464" | "application/json" | "application/x-ndjson" | "json" | "ndjson" => {
10769 Some("json")
10770 }
10771 "application/parquet" | "application/vnd.apache.parquet" | "parquet" => Some("parquet"),
10772 "application/vnd.apache.arrow.file" | "arrow" | "ipc" | "feather" => Some("arrow"),
10773 "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" | "xlsx" => {
10774 Some("xlsx")
10775 }
10776 _ => None,
10777 }
10778}
10779
10780fn datafile_storage_uri_path(lake_root: &str, storage_uri: &str) -> Result<PathBuf, AppError> {
10781 if let Some(relative) = storage_uri.strip_prefix("lake://") {
10782 return Ok(Path::new(lake_root).join(relative));
10783 }
10784 if storage_uri.starts_with("file://") {
10785 return file_uri_to_path(storage_uri);
10786 }
10787 Err(AppError::Validation(
10788 "datafile storage URI must use file:// or lake://".to_string(),
10789 ))
10790}
10791
10792fn file_uri_to_path(uri: &str) -> Result<PathBuf, AppError> {
10793 let rest = uri
10794 .strip_prefix("file://")
10795 .ok_or_else(|| AppError::Validation("datafile URI must use file://".to_string()))?;
10796 let path = if let Some(path) = rest.strip_prefix("localhost") {
10797 path
10798 } else if rest.starts_with('/') || (cfg!(windows) && rest.as_bytes().get(1) == Some(&b':')) {
10799 rest
10800 } else {
10801 return Err(AppError::Validation(
10802 "file URI must refer to a local absolute path".to_string(),
10803 ));
10804 };
10805 let decoded = percent_decode_path(path)?;
10806 if cfg!(windows) {
10807 let mut value = decoded.replace('/', "\\");
10808 if value.starts_with('\\') && value.len() > 3 && value.as_bytes()[2] == b':' {
10809 value.remove(0);
10810 }
10811 Ok(PathBuf::from(value))
10812 } else {
10813 Ok(PathBuf::from(decoded))
10814 }
10815}
10816
10817fn percent_decode_path(path: &str) -> Result<String, AppError> {
10818 let mut decoded = Vec::with_capacity(path.len());
10819 let bytes = path.as_bytes();
10820 let mut index = 0;
10821 while index < bytes.len() {
10822 if bytes[index] == b'%' {
10823 if index + 2 >= bytes.len() {
10824 return Err(AppError::Validation(
10825 "file URI contains incomplete percent escape".to_string(),
10826 ));
10827 }
10828 let high = percent_hex_value(bytes[index + 1])?;
10829 let low = percent_hex_value(bytes[index + 2])?;
10830 decoded.push((high << 4) | low);
10831 index += 3;
10832 } else {
10833 decoded.push(bytes[index]);
10834 index += 1;
10835 }
10836 }
10837 String::from_utf8(decoded)
10838 .map_err(|error| AppError::Validation(format!("file URI path is not UTF-8: {error}")))
10839}
10840
10841fn percent_hex_value(byte: u8) -> Result<u8, AppError> {
10842 match byte {
10843 b'0'..=b'9' => Ok(byte - b'0'),
10844 b'a'..=b'f' => Ok(byte - b'a' + 10),
10845 b'A'..=b'F' => Ok(byte - b'A' + 10),
10846 _ => Err(AppError::Validation(format!(
10847 "file URI contains invalid percent escape byte {}",
10848 byte as char
10849 ))),
10850 }
10851}
10852
10853fn dataset_file_read_options(
10854 header: Option<bool>,
10855 delimiter: Option<char>,
10856 null_strings: Vec<String>,
10857 json_format: Option<String>,
10858 sheet: Option<String>,
10859) -> DatasetFileReadOptions {
10860 let defaults = DatasetFileReadOptions::default();
10861 DatasetFileReadOptions {
10862 header: header.unwrap_or(defaults.header),
10863 delimiter: delimiter.unwrap_or(defaults.delimiter),
10864 null_strings,
10865 json_format: json_format.unwrap_or(defaults.json_format),
10866 sheet,
10867 }
10868}
10869
10870struct RedcapArtifactReader<'a> {
10871 file: std::fs::File,
10872 limits: Option<&'a ahri_tre_lake::UploadValidation>,
10873}
10874impl std::io::Read for RedcapArtifactReader<'_> {
10875 fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
10876 if let Some(limits) = self.limits {
10877 limits
10878 .check_deadline()
10879 .map_err(|_| std::io::Error::other("REDCap acquisition interrupted"))?;
10880 }
10881 std::io::Read::read(&mut self.file, out)
10882 }
10883}
10884
10885struct RedcapArtifactInput {
10886 suffix: &'static str,
10887 path: PathBuf,
10888 edam_format: &'static str,
10889 description: String,
10890}
10891
10892struct RedcapStudyResolveContext<'a> {
10893 study_id: Option<ahri_tre_types::StudyId>,
10894 domain_id: ahri_tre_types::DomainId,
10895 study_name: Option<&'a ahri_tre_types::NcName>,
10896 agent_instructions: Option<&'a String>,
10897 ingest_transformation: &'a ahri_tre_types::NewTransformationRecord,
10898}
10899
10900struct RedcapArtifactStageContext<'a> {
10901 limits: Option<&'a ahri_tre_lake::UploadValidation>,
10902 operator: Option<ahri_tre_pgmeta::governance::GovernanceMaintenance>,
10903 classification: &'a ahri_tre_types::IngestClassification,
10904 study_id: ahri_tre_types::StudyId,
10905 lake_root: &'a str,
10906 asset_prefix: &'a str,
10907 compress: bool,
10908 encrypt: bool,
10909 agent_instructions: Option<&'a String>,
10910 version_note: Option<&'a String>,
10911 created_by: Option<&'a String>,
10912}
10913
10914type RedcapFormsResult = (
10915 Vec<DatasetMaterialization>,
10916 Vec<String>,
10917 Vec<ahri_tre_protocol::ingest::RedcapFormOutcome>,
10918);
10919
10920struct RedcapMaterializeOptions<'a> {
10921 limits: Option<&'a ahri_tre_lake::UploadValidation>,
10922 no_register_dictionary: bool,
10923 dataset_prefix: Option<&'a ahri_tre_types::NcName>,
10924 agent_instructions: Option<&'a String>,
10925 version_note: Option<&'a String>,
10926 transform_transformation: &'a ahri_tre_types::NewTransformationRecord,
10927 force_metadata_updates: bool,
10928}
10929
10930#[derive(Clone)]
10931struct StagedRedcapArtifact {
10932 suffix: String,
10933 asset: ahri_tre_types::AssetRecord,
10934 version: ahri_tre_types::AssetVersionRecord,
10935 datafile: ahri_tre_types::DataFileRecord,
10936 source_size_bytes: u64,
10937 stored_size_bytes: u64,
10938}
10939
10940fn redcap_asset_prefix(project_info: &RedcapProjectInfo) -> Result<String, AppError> {
10941 let value = format!(
10942 "redcap_{}",
10943 sanitize_ncname_fragment(&project_info.project_id)
10944 );
10945 ahri_tre_types::NcName::parse(&value).map_err(|error| {
10946 AppError::Validation(format!("REDCap asset prefix is invalid: {error}"))
10947 })?;
10948 Ok(value)
10949}
10950
10951fn redcap_domain_name(project_info: &RedcapProjectInfo) -> String {
10952 format!(
10953 "redcap_{}",
10954 sanitize_ncname_fragment(&project_info.project_id)
10955 )
10956}
10957
10958fn redcap_dataset_name(
10959 project_info: &RedcapProjectInfo,
10960 form: &RedcapForm,
10961 prefix: Option<&ahri_tre_types::NcName>,
10962) -> Result<ahri_tre_types::NcName, AppError> {
10963 let prefix = prefix
10964 .map(|value| value.as_str().to_string())
10965 .unwrap_or_else(|| {
10966 format!(
10967 "redcap_{}",
10968 sanitize_ncname_fragment(&project_info.project_id)
10969 )
10970 });
10971 ahri_tre_types::NcName::parse(format!(
10972 "{}_{}",
10973 prefix,
10974 sanitize_ncname_fragment(&form.name)
10975 ))
10976 .map_err(|error| AppError::Validation(format!("REDCap dataset name is invalid: {error}")))
10977}
10978
10979fn redcap_dataset_variable_registrations(
10980 domain_id: ahri_tre_types::DomainId,
10981 fields: &[RedcapVariableSpec],
10982 synthetic_columns: &[RedcapSyntheticColumn],
10983) -> Result<(Vec<DatasetVariableRegistrationInput>, Vec<String>), AppError> {
10984 let mut registrations = Vec::with_capacity(fields.len() + synthetic_columns.len());
10985 let mut warnings = Vec::new();
10986 for field in fields {
10987 let (registration, mut field_warnings) = redcap_variable_registration(domain_id, field)?;
10988 registrations.push(registration);
10989 warnings.append(&mut field_warnings);
10990 }
10991 for column in synthetic_columns {
10992 let (registration, mut column_warnings) = redcap_synthetic_registration(domain_id, column)?;
10993 registrations.push(registration);
10994 warnings.append(&mut column_warnings);
10995 }
10996 Ok((registrations, warnings))
10997}
10998
10999fn redcap_variable_registration(
11000 domain_id: ahri_tre_types::DomainId,
11001 field: &RedcapVariableSpec,
11002) -> Result<(DatasetVariableRegistrationInput, Vec<String>), AppError> {
11003 let name = ahri_tre_types::NcName::parse(&field.field_name).map_err(|error| {
11004 AppError::Validation(format!(
11005 "REDCap field {} is not a valid variable name: {error}",
11006 field.field_name
11007 ))
11008 })?;
11009 let vocabulary = if field.choices.is_empty() {
11010 None
11011 } else {
11012 Some(ahri_tre_types::NewVocabularyRecord {
11013 domain_id,
11014 name: ahri_tre_types::NcName::parse(format!("redcap_{}", field.field_name)).map_err(
11015 |error| {
11016 AppError::Validation(format!(
11017 "REDCap vocabulary name for {} is invalid: {error}",
11018 field.field_name
11019 ))
11020 },
11021 )?,
11022 description: Some(format!("REDCap choices for {}", field.field_name)),
11023 })
11024 };
11025 let (vocabulary_items, warnings) = redcap_vocabulary_items(
11026 &format!("REDCap field {}", field.field_name),
11027 &field.choices,
11028 );
11029 Ok((
11030 DatasetVariableRegistrationInput::New(NewDatasetVariableRegistration {
11031 variable: ahri_tre_types::NewVariableRecord {
11032 domain_id,
11033 name,
11034 value_type_id: field.value_type_id,
11035 value_format: field.value_format.clone(),
11036 vocabulary_id: None,
11037 key_role: if field.is_record_id {
11038 ahri_tre_types::KeyRole::Record
11039 } else {
11040 ahri_tre_types::KeyRole::None
11041 },
11042 description: field.label.clone(),
11043 note: field.note.clone(),
11044 ontology_namespace: None,
11045 ontology_class: None,
11046 },
11047 row_role: Some(if field.is_record_id {
11048 ahri_tre_types::KeyRole::Record
11049 } else {
11050 ahri_tre_types::KeyRole::None
11051 }),
11052 vocabulary,
11053 vocabulary_items,
11054 source_column_name: None,
11055 }),
11056 warnings,
11057 ))
11058}
11059
11060fn redcap_synthetic_registration(
11061 domain_id: ahri_tre_types::DomainId,
11062 column: &RedcapSyntheticColumn,
11063) -> Result<(DatasetVariableRegistrationInput, Vec<String>), AppError> {
11064 let name = ahri_tre_types::NcName::parse(&column.output_column).map_err(|error| {
11065 AppError::Validation(format!(
11066 "REDCap synthetic column {} is not a valid variable name: {error}",
11067 column.output_column
11068 ))
11069 })?;
11070 let choices = column
11071 .choices
11072 .iter()
11073 .map(|choice| RedcapChoice {
11074 value: choice.value,
11075 code: choice.redcap_value.clone(),
11076 description: None,
11077 redcap_value: choice.redcap_value.clone(),
11078 })
11079 .collect::<Vec<_>>();
11080 let vocabulary = if choices.is_empty() {
11081 None
11082 } else {
11083 Some(ahri_tre_types::NewVocabularyRecord {
11084 domain_id,
11085 name: name.clone(),
11086 description: Some(format!(
11087 "REDCap metadata values for {}",
11088 column.source_column
11089 )),
11090 })
11091 };
11092 let (vocabulary_items, warnings) = redcap_vocabulary_items(
11093 &format!("REDCap synthetic column {}", column.output_column),
11094 &choices,
11095 );
11096 Ok((
11097 DatasetVariableRegistrationInput::New(NewDatasetVariableRegistration {
11098 variable: ahri_tre_types::NewVariableRecord {
11099 domain_id,
11100 name,
11101 value_type_id: column.value_type_id,
11102 value_format: None,
11103 vocabulary_id: None,
11104 key_role: ahri_tre_types::KeyRole::None,
11105 description: Some(format!("REDCap {}", column.source_column)),
11106 note: Some("Synthetic REDCap EAV metadata column".to_string()),
11107 ontology_namespace: None,
11108 ontology_class: None,
11109 },
11110 row_role: Some(ahri_tre_types::KeyRole::None),
11111 vocabulary,
11112 vocabulary_items,
11113 source_column_name: None,
11114 }),
11115 warnings,
11116 ))
11117}
11118
11119fn redcap_vocabulary_items(
11120 context: &str,
11121 choices: &[RedcapChoice],
11122) -> (Vec<ahri_tre_types::NewVocabularyItemRecord>, Vec<String>) {
11123 let mut items: Vec<ahri_tre_types::NewVocabularyItemRecord> = Vec::with_capacity(choices.len());
11124 let mut warnings = Vec::new();
11125 for choice in choices {
11126 let (code, code_warning) = normalize_redcap_vocabulary_item_code(context, &choice.code);
11127 if let Some(warning) = code_warning {
11128 warnings.push(warning);
11129 }
11130 let candidate = ahri_tre_types::NewVocabularyItemRecord {
11131 value: choice.value,
11132 code,
11133 description: choice.description.clone(),
11134 };
11135 match items
11136 .iter()
11137 .find(|existing| existing.value == candidate.value || existing.code == candidate.code)
11138 {
11139 Some(existing)
11140 if existing.value == candidate.value
11141 && existing.code == candidate.code
11142 && existing.description == candidate.description => {}
11143 Some(existing) => {
11144 warnings.push(redcap_vocabulary_item_conflict_warning(
11145 context, existing, &candidate,
11146 ));
11147 }
11148 None => items.push(candidate),
11149 }
11150 }
11151 (items, warnings)
11152}
11153
11154const REDCAP_VOCABULARY_ITEM_CODE_MAX_LEN: usize = 80;
11155const REDCAP_VOCABULARY_ITEM_CODE_HASH_LEN: usize = 8;
11156
11157fn normalize_redcap_vocabulary_item_code(context: &str, code: &str) -> (String, Option<String>) {
11158 let normalized = bounded_redcap_vocabulary_item_code(code);
11159 if normalized == code {
11160 return (normalized, None);
11161 }
11162 (
11163 normalized.clone(),
11164 Some(format!(
11165 "{context} has REDCap choice code length {} exceeding the metadata limit {}; storing {} instead",
11166 code.chars().count(),
11167 REDCAP_VOCABULARY_ITEM_CODE_MAX_LEN,
11168 normalized,
11169 )),
11170 )
11171}
11172
11173fn bounded_redcap_vocabulary_item_code(code: &str) -> String {
11174 if code.chars().count() <= REDCAP_VOCABULARY_ITEM_CODE_MAX_LEN {
11175 return code.to_string();
11176 }
11177
11178 let suffix = format!("_{:08x}", redcap_vocabulary_item_code_hash(code));
11179 let prefix_len = REDCAP_VOCABULARY_ITEM_CODE_MAX_LEN
11180 .saturating_sub(REDCAP_VOCABULARY_ITEM_CODE_HASH_LEN + 1);
11181 let prefix: String = code.chars().take(prefix_len).collect();
11182 format!("{prefix}{suffix}")
11183}
11184
11185fn redcap_vocabulary_item_code_hash(code: &str) -> u32 {
11186 let mut hash = 0x811c_9dc5_u32;
11187 for byte in code.as_bytes() {
11188 hash ^= u32::from(*byte);
11189 hash = hash.wrapping_mul(0x0100_0193);
11190 }
11191 hash
11192}
11193
11194fn redcap_vocabulary_item_conflict_warning(
11195 context: &str,
11196 existing: &ahri_tre_types::NewVocabularyItemRecord,
11197 candidate: &ahri_tre_types::NewVocabularyItemRecord,
11198) -> String {
11199 if existing.code == candidate.code && existing.value != candidate.value {
11200 return format!(
11201 "{context} has conflicting REDCap choice code {} with values {} and {}; keeping the first value {}",
11202 existing.code, existing.value, candidate.value, existing.value
11203 );
11204 }
11205 if existing.value == candidate.value && existing.code != candidate.code {
11206 return format!(
11207 "{context} has conflicting REDCap choice value {} with codes {} and {}; keeping the first code {}",
11208 existing.value, existing.code, candidate.code, existing.code
11209 );
11210 }
11211 format!(
11212 "{context} has conflicting REDCap choice metadata for value {} and code {}; keeping the first description",
11213 existing.value, existing.code
11214 )
11215}
11216
11217enum VocabularyItemReconciliation {
11218 ExactMatch,
11219 Conflict(usize),
11220 Missing,
11221}
11222
11223struct VocabularyItemCandidate<'a> {
11224 vocabulary_item_id: Option<ahri_tre_types::VocabularyItemId>,
11225 value: i32,
11226 code: &'a str,
11227 description: Option<&'a str>,
11228}
11229
11230fn vocabulary_item_candidate<'a>(
11231 vocabulary_item_id: Option<ahri_tre_types::VocabularyItemId>,
11232 value: i32,
11233 code: &'a str,
11234 description: Option<&'a str>,
11235) -> VocabularyItemCandidate<'a> {
11236 VocabularyItemCandidate {
11237 vocabulary_item_id,
11238 value,
11239 code,
11240 description,
11241 }
11242}
11243
11244struct SeenVocabularyItem {
11245 vocabulary_item_id: Option<ahri_tre_types::VocabularyItemId>,
11246 value: i32,
11247 code: String,
11248 description: Option<String>,
11249 pending_create_index: Option<usize>,
11250}
11251
11252impl SeenVocabularyItem {
11253 fn from_record(record: &ahri_tre_types::VocabularyItemRecord) -> Self {
11254 Self {
11255 vocabulary_item_id: Some(record.vocabulary_item_id),
11256 value: record.value,
11257 code: record.code.clone(),
11258 description: record.description.clone(),
11259 pending_create_index: None,
11260 }
11261 }
11262
11263 fn from_pending_create(
11264 pending_create_index: usize,
11265 item: &ahri_tre_types::NewVocabularyItemRecord,
11266 ) -> Self {
11267 Self {
11268 vocabulary_item_id: None,
11269 value: item.value,
11270 code: item.code.clone(),
11271 description: item.description.clone(),
11272 pending_create_index: Some(pending_create_index),
11273 }
11274 }
11275}
11276
11277fn classify_vocabulary_item(
11278 seen_items: &[SeenVocabularyItem],
11279 candidate: &VocabularyItemCandidate<'_>,
11280) -> VocabularyItemReconciliation {
11281 let Some(existing_index) = seen_items.iter().position(|existing| {
11282 candidate
11283 .vocabulary_item_id
11284 .is_some_and(|id| existing.vocabulary_item_id == Some(id))
11285 || existing.value == candidate.value
11286 || existing.code == candidate.code
11287 }) else {
11288 return VocabularyItemReconciliation::Missing;
11289 };
11290 let existing = &seen_items[existing_index];
11291 if existing.value == candidate.value
11292 && existing.code == candidate.code
11293 && existing.description.as_deref() == candidate.description
11294 {
11295 VocabularyItemReconciliation::ExactMatch
11296 } else {
11297 VocabularyItemReconciliation::Conflict(existing_index)
11298 }
11299}
11300
11301fn build_vocabulary_item_record(
11302 vocabulary_item_id: ahri_tre_types::VocabularyItemId,
11303 vocabulary_id: ahri_tre_types::VocabularyId,
11304 value: i32,
11305 code: String,
11306 description: Option<String>,
11307) -> ahri_tre_types::VocabularyItemRecord {
11308 ahri_tre_types::VocabularyItemRecord {
11309 vocabulary_item_id,
11310 vocabulary_id,
11311 value,
11312 code,
11313 description,
11314 }
11315}
11316
11317fn vocabulary_item_conflict_error(
11318 requested_id: Option<ahri_tre_types::VocabularyItemId>,
11319 existing: &SeenVocabularyItem,
11320 candidate: &VocabularyItemCandidate<'_>,
11321) -> AppError {
11322 let requested = requested_id
11323 .or(candidate.vocabulary_item_id)
11324 .map(|id| format!("vocabulary item {}", id.0))
11325 .unwrap_or_else(|| "vocabulary item candidate".to_string());
11326 AppError::Validation(format!(
11327 "{requested} is {}:{}, not {}:{}; set force_metadata_updates to update metadata",
11328 existing.value, existing.code, candidate.value, candidate.code
11329 ))
11330}
11331
11332fn redcap_lake_field(field: &RedcapVariableSpec) -> RedcapEavField {
11333 RedcapEavField {
11334 name: field.field_name.clone(),
11335 value_type_id: field.value_type_id,
11336 value_format: field.value_format.clone(),
11337 choices: field
11338 .choices
11339 .iter()
11340 .map(|choice| RedcapEavChoice {
11341 redcap_value: choice.redcap_value.clone(),
11342 value: choice.value,
11343 })
11344 .collect(),
11345 }
11346}
11347
11348fn redcap_transform_record(
11349 template: &ahri_tre_types::NewTransformationRecord,
11350 form_name: &str,
11351) -> ahri_tre_types::NewTransformationRecord {
11352 let mut record = template.clone();
11353 record.description = format!("{} ({form_name})", template.description);
11354 record
11355}
11356
11357fn redcap_eav_present_fields(
11358 connection: &duckdb::Connection,
11359 eav_path: &Path,
11360 field_names: &[&str],
11361) -> Result<std::collections::BTreeSet<String>, AppError> {
11362 if field_names.is_empty() {
11363 return Ok(std::collections::BTreeSet::new());
11364 }
11365 let path = sql_literal(&eav_path.display().to_string());
11366 let fields = field_names
11367 .iter()
11368 .map(|value| sql_literal(value))
11369 .collect::<Vec<_>>()
11370 .join(", ");
11371 ensure_redcap_eav_columns(connection, eav_path, &["field_name", "value"])?;
11372 let sql = format!(
11373 "SELECT DISTINCT field_name FROM {} WHERE field_name IN ({fields}) ORDER BY field_name;",
11374 redcap_eav_scan_sql(&path)
11375 );
11376 let mut statement = connection
11377 .prepare(&sql)
11378 .map_err(|error| infrastructure_error("inspect REDCap EAV fields", error))?;
11379 let rows = statement
11380 .query_map([], |row| row.get::<_, String>(0))
11381 .map_err(|error| infrastructure_error("read REDCap EAV fields", error))?;
11382 let mut values = std::collections::BTreeSet::new();
11383 for row in rows {
11384 values.insert(row.map_err(|error| infrastructure_error("read REDCap EAV field", error))?);
11385 }
11386 Ok(values)
11387}
11388
11389fn redcap_synthetic_columns(
11390 connection: &duckdb::Connection,
11391 eav_path: &Path,
11392 dataset_name: &str,
11393) -> Result<Vec<RedcapSyntheticColumn>, AppError> {
11394 let mut columns = Vec::new();
11395 let event_values =
11396 redcap_eav_distinct_column_values(connection, eav_path, "redcap_event_name")?;
11397 if !event_values.is_empty() {
11398 columns.push(RedcapSyntheticColumn {
11399 source_column: "redcap_event_name".to_string(),
11400 output_column: redcap_synthetic_output_column(dataset_name, "redcap_event"),
11401 value_type_id: ahri_tre_types::ValueTypeId(7),
11402 choices: sequential_redcap_choices(event_values),
11403 });
11404 }
11405 let instrument_values =
11406 redcap_eav_distinct_column_values(connection, eav_path, "redcap_repeat_instrument")?;
11407 if !instrument_values.is_empty() {
11408 columns.push(RedcapSyntheticColumn {
11409 source_column: "redcap_repeat_instrument".to_string(),
11410 output_column: redcap_synthetic_output_column(dataset_name, "redcap_repeat_instrument"),
11411 value_type_id: ahri_tre_types::ValueTypeId(7),
11412 choices: sequential_redcap_choices(instrument_values),
11413 });
11414 }
11415 if redcap_eav_has_column(connection, eav_path, "redcap_repeat_instance")? {
11416 columns.push(RedcapSyntheticColumn {
11417 source_column: "redcap_repeat_instance".to_string(),
11418 output_column: redcap_synthetic_output_column(dataset_name, "redcap_repeat_instance"),
11419 value_type_id: ahri_tre_types::ValueTypeId(1),
11420 choices: Vec::new(),
11421 });
11422 }
11423 Ok(columns)
11424}
11425
11426fn redcap_synthetic_output_column(dataset_name: &str, suffix: &str) -> String {
11427 const MAX_IDENTIFIER_LEN: usize = 63;
11428
11429 let full_name = format!("{dataset_name}_{suffix}");
11430 if full_name.len() <= MAX_IDENTIFIER_LEN {
11431 return full_name;
11432 }
11433
11434 let hash = fnv1a64_hex(&full_name);
11435 let reserved = suffix.len() + hash.len() + 2;
11436 let available_prefix = MAX_IDENTIFIER_LEN.saturating_sub(reserved);
11437 let prefix = dataset_name
11438 .chars()
11439 .take(available_prefix)
11440 .collect::<String>()
11441 .trim_end_matches('_')
11442 .to_string();
11443
11444 format!("{prefix}_{suffix}_{hash}")
11445}
11446
11447fn fnv1a64_hex(value: &str) -> String {
11448 let mut hash: u64 = 0xcbf29ce484222325;
11449 for byte in value.as_bytes() {
11450 hash ^= u64::from(*byte);
11451 hash = hash.wrapping_mul(0x100000001b3);
11452 }
11453 format!("{hash:08x}")
11454}
11455
11456fn sequential_redcap_choices(values: Vec<String>) -> Vec<RedcapEavChoice> {
11457 values
11458 .into_iter()
11459 .enumerate()
11460 .map(|(index, value)| RedcapEavChoice {
11461 redcap_value: value,
11462 value: i32::try_from(index + 1).unwrap_or(i32::MAX),
11463 })
11464 .collect()
11465}
11466
11467fn redcap_eav_distinct_column_values(
11468 connection: &duckdb::Connection,
11469 eav_path: &Path,
11470 column: &str,
11471) -> Result<Vec<String>, AppError> {
11472 if !redcap_eav_has_column(connection, eav_path, column)? {
11473 return Ok(Vec::new());
11474 }
11475 let path = sql_literal(&eav_path.display().to_string());
11476 let column_sql = sql_identifier(column);
11477 let sql = format!(
11478 "SELECT DISTINCT CAST({column_sql} AS VARCHAR) FROM {} WHERE {column_sql} IS NOT NULL AND length(trim(CAST({column_sql} AS VARCHAR))) > 0 ORDER BY 1 LIMIT 4097;",
11479 redcap_eav_scan_sql(&path)
11480 );
11481 let mut statement = connection
11482 .prepare(&sql)
11483 .map_err(|error| infrastructure_error("inspect REDCap EAV metadata values", error))?;
11484 let rows = statement
11485 .query_map([], |row| row.get::<_, String>(0))
11486 .map_err(|error| infrastructure_error("read REDCap EAV metadata values", error))?;
11487 let mut values = Vec::new();
11488 for row in rows {
11489 values.push(row.map_err(|error| infrastructure_error("read REDCap EAV value", error))?);
11490 if values.len() > 4096 {
11491 return Err(AppError::Validation(
11492 "REDCap synthetic vocabulary exceeds limit".into(),
11493 ));
11494 }
11495 }
11496 Ok(values)
11497}
11498
11499fn redcap_eav_has_column(
11500 connection: &duckdb::Connection,
11501 eav_path: &Path,
11502 column: &str,
11503) -> Result<bool, AppError> {
11504 let columns = redcap_eav_columns(connection, eav_path)?;
11505 Ok(columns.iter().any(|value| value == column))
11506}
11507
11508fn ensure_redcap_eav_columns(
11509 connection: &duckdb::Connection,
11510 eav_path: &Path,
11511 required: &[&str],
11512) -> Result<(), AppError> {
11513 let columns = redcap_eav_columns(connection, eav_path)?;
11514 let missing = required
11515 .iter()
11516 .filter(|column| !columns.iter().any(|value| value == **column))
11517 .copied()
11518 .collect::<Vec<_>>();
11519 if !missing.is_empty() {
11520 return Err(AppError::Infrastructure(format!(
11521 "inspect REDCap EAV fields: expected REDCap EAV CSV columns {} in {}, got {}",
11522 missing.join(", "),
11523 eav_path.display(),
11524 if columns.is_empty() {
11525 "<none>".to_string()
11526 } else {
11527 columns.join(", ")
11528 }
11529 )));
11530 }
11531 Ok(())
11532}
11533
11534fn redcap_eav_columns(
11535 connection: &duckdb::Connection,
11536 eav_path: &Path,
11537) -> Result<Vec<String>, AppError> {
11538 let path = sql_literal(&eav_path.display().to_string());
11539 let sql = format!("DESCRIBE SELECT * FROM {};", redcap_eav_scan_sql(&path));
11540 let mut statement = connection
11541 .prepare(&sql)
11542 .map_err(|error| infrastructure_error("inspect REDCap EAV columns", error))?;
11543 let rows = statement
11544 .query_map([], |row| row.get::<_, String>(0))
11545 .map_err(|error| infrastructure_error("read REDCap EAV columns", error))?;
11546 let mut columns = Vec::new();
11547 for row in rows {
11548 columns.push(row.map_err(|error| infrastructure_error("read REDCap EAV column", error))?);
11549 }
11550 Ok(columns)
11551}
11552
11553fn redcap_eav_scan_sql(path: &str) -> String {
11554 format!("read_csv_auto({path}, header=true, delim=',', all_varchar=true)")
11555}
11556
11557#[derive(Debug, Clone, Copy)]
11558enum DatasetOutputIntent {
11559 Create,
11560 Replace,
11561 Version(i32, i32, i32),
11562}
11563impl DatasetOutputIntent {
11564 fn parse(replace: bool, version: Option<&str>) -> Result<Self, AppError> {
11565 match (replace, version) {
11566 (false, None) => Ok(Self::Create),
11567 (true, None) => Ok(Self::Replace),
11568 (false, Some(version)) => {
11569 let parts = version
11570 .split('.')
11571 .map(str::parse::<i32>)
11572 .collect::<Result<Vec<_>, _>>()
11573 .map_err(|_| {
11574 AppError::Validation("Output version requires major.minor.patch".into())
11575 })?;
11576 match parts.as_slice() {
11577 [major, minor, patch] if parts.iter().all(|value| *value >= 0) => {
11578 Ok(Self::Version(*major, *minor, *patch))
11579 }
11580 _ => Err(AppError::Validation(
11581 "Output version requires major.minor.patch".into(),
11582 )),
11583 }
11584 }
11585 _ => Err(AppError::Validation(
11586 "Replacement and an explicit version are mutually exclusive".into(),
11587 )),
11588 }
11589 }
11590}
11591
11592struct PrepareDatasetVersionRequest {
11593 output_intent: Option<DatasetOutputIntent>,
11594 inherit_risk: bool,
11595 pub classification: ahri_tre_types::IngestClassification,
11596 study_id: ahri_tre_types::StudyId,
11597 dataset_asset_id: Option<ahri_tre_types::AssetId>,
11598 dataset_name: ahri_tre_types::NcName,
11599 dataset_version_id: Option<ahri_tre_types::VersionId>,
11600 description: Option<String>,
11601 agent_instructions: Option<String>,
11602 version_note: Option<String>,
11603 created_by: Option<String>,
11604}
11605
11606#[derive(Debug, Clone)]
11607struct PreparedDatasetVersion {
11608 ordinary_upload_source: Option<ahri_tre_types::VersionId>,
11609 inherit_risk: bool,
11610 pub classification: ahri_tre_types::IngestClassification,
11611 asset: ahri_tre_types::AssetRecord,
11612 version: ahri_tre_types::AssetVersionRecord,
11613 asset_persisted: bool,
11614 version_persisted: bool,
11615 asset_tags: Vec<String>,
11616 version_tags: Vec<String>,
11617}
11618
11619type DatasetAdmissionRecords = (
11620 AssetVersionCatalog,
11621 ahri_tre_types::DatasetRecord,
11622 ahri_tre_types::TransformationLineageRecord,
11623 Vec<RegisteredDatasetVariable>,
11624);
11625struct DatasetAdmissionMetadata<'a> {
11626 dataset: ahri_tre_types::DatasetRecord,
11627 transformation: &'a ahri_tre_types::NewTransformationRecord,
11628 input_version_ids: &'a [ahri_tre_types::VersionId],
11629 registration: DatasetMetadataRegistration<'a>,
11630}
11631
11632struct DatasetMetadataRegistration<'a> {
11633 redcap_dictionary: Option<(ahri_tre_types::DomainId, &'a [RedcapForm])>,
11634 variable_registrations: Vec<DatasetVariableRegistrationInput>,
11635 loaded_column_names: Option<&'a [String]>,
11636 force_metadata_updates: bool,
11637}
11638
11639fn derived_source_file_asset_name(dataset_name: &ahri_tre_types::NcName) -> ahri_tre_types::NcName {
11640 ahri_tre_types::NcName::parse(format!("{}_source_file", dataset_name.as_str()))
11641 .expect("derived source file asset name should remain an NcName")
11642}
11643
11644#[derive(Debug, Clone)]
11645enum DatasetVariableRegistrationInput {
11646 Explicit(DatasetVariableRegistrationRequest),
11647 New(NewDatasetVariableRegistration),
11648}
11649
11650impl From<DatasetVariableRegistrationRequest> for DatasetVariableRegistrationInput {
11651 fn from(registration: DatasetVariableRegistrationRequest) -> Self {
11652 Self::Explicit(registration)
11653 }
11654}
11655
11656#[derive(Debug, Clone)]
11657struct NewDatasetVariableRegistration {
11658 variable: ahri_tre_types::NewVariableRecord,
11659 row_role: Option<ahri_tre_types::KeyRole>,
11660 vocabulary: Option<ahri_tre_types::NewVocabularyRecord>,
11661 vocabulary_items: Vec<ahri_tre_types::NewVocabularyItemRecord>,
11662 source_column_name: Option<String>,
11663}
11664
11665fn sql_identifier(value: &str) -> String {
11666 format!("\"{}\"", value.replace('"', "\"\""))
11667}
11668
11669fn sql_literal(value: &str) -> String {
11670 format!("'{}'", value.replace('\'', "''"))
11671}
11672
11673fn first_registration_domain(
11674 registrations: &[DatasetVariableRegistrationInput],
11675) -> Option<ahri_tre_types::DomainId> {
11676 registrations
11677 .first()
11678 .map(|registration| match registration {
11679 DatasetVariableRegistrationInput::Explicit(registration) => {
11680 registration.variable.domain_id
11681 }
11682 DatasetVariableRegistrationInput::New(registration) => registration.variable.domain_id,
11683 })
11684}
11685
11686fn value_type_id_for_duckdb_type(duckdb_type: &str) -> ahri_tre_types::ValueTypeId {
11687 let lowered = duckdb_type.to_ascii_lowercase();
11688 if lowered.contains("int") {
11689 ahri_tre_types::ValueTypeId(1)
11690 } else if lowered.contains("float")
11691 || lowered.contains("double")
11692 || lowered.contains("real")
11693 || lowered.contains("decimal")
11694 || lowered.contains("numeric")
11695 {
11696 ahri_tre_types::ValueTypeId(2)
11697 } else if lowered == "date" {
11698 ahri_tre_types::ValueTypeId(4)
11699 } else if lowered.contains("timestamp") || lowered.contains("datetime") {
11700 ahri_tre_types::ValueTypeId(5)
11701 } else if lowered == "time" || lowered.starts_with("time(") {
11702 ahri_tre_types::ValueTypeId(6)
11703 } else if lowered.contains("bool") {
11704 ahri_tre_types::ValueTypeId(1)
11705 } else {
11706 ahri_tre_types::ValueTypeId(3)
11707 }
11708}
11709
11710fn is_string_like_duckdb_type(duckdb_type: &str) -> bool {
11711 let lowered = duckdb_type.to_ascii_lowercase();
11712 lowered.contains("char")
11713 || lowered.contains("varchar")
11714 || lowered.contains("string")
11715 || lowered.contains("text")
11716}
11717
11718fn field_metadata_proposal_is_rich(proposal: &FieldMetadataProposal) -> bool {
11719 proposal.variable_name.is_some()
11720 || proposal.description.is_some()
11721 || proposal.value_type.is_some()
11722 || proposal.value_type_id.is_some()
11723 || proposal.vocabulary_name.is_some()
11724 || proposal.vocabulary_description.is_some()
11725 || !proposal.vocabulary_items.is_empty()
11726}
11727
11728fn profile_string_like_column_vocabularies(
11729 connection: &duckdb::Connection,
11730 relation: &DatasetTableRelation,
11731 columns: &[ImportedColumn],
11732 sample_rows: Option<usize>,
11733 vocabulary_threshold: Option<usize>,
11734 null_strings: &[String],
11735) -> Result<BTreeMap<String, Vec<String>>, AppError> {
11736 let threshold = vocabulary_threshold.unwrap_or(250);
11737 if threshold == 0 {
11738 return Ok(BTreeMap::new());
11739 }
11740 let mut profiled = BTreeMap::new();
11741 for column in columns {
11742 if !is_string_like_duckdb_type(&column.duckdb_type) {
11743 continue;
11744 }
11745 let values = profile_distinct_string_values(
11746 connection,
11747 relation,
11748 column.name.as_str(),
11749 sample_rows,
11750 threshold,
11751 null_strings,
11752 )?;
11753 if !values.is_empty() && values.len() <= threshold {
11754 profiled.insert(column.name.clone(), values);
11755 }
11756 }
11757 Ok(profiled)
11758}
11759
11760fn profile_distinct_string_values(
11761 connection: &duckdb::Connection,
11762 relation: &DatasetTableRelation,
11763 column_name: &str,
11764 sample_rows: Option<usize>,
11765 threshold: usize,
11766 null_strings: &[String],
11767) -> Result<Vec<String>, AppError> {
11768 let relation_sql = format!(
11769 "{}.{}",
11770 quote_sql_identifier(&relation.schema_name),
11771 quote_sql_identifier(&relation.table_name)
11772 );
11773 let column_sql = quote_sql_identifier(column_name);
11774 let sampled_sql = match sample_rows {
11775 Some(limit) => format!("SELECT {column_sql} AS value FROM {relation_sql} LIMIT {limit}"),
11776 None => format!("SELECT {column_sql} AS value FROM {relation_sql}"),
11777 };
11778 let null_string_filter = if null_strings.is_empty() {
11779 String::new()
11780 } else {
11781 let values = null_strings
11782 .iter()
11783 .map(|value| sql_string_literal(value))
11784 .collect::<Vec<_>>()
11785 .join(", ");
11786 format!(" AND CAST(value AS VARCHAR) NOT IN ({values})")
11787 };
11788 let sql = format!(
11789 "SELECT DISTINCT CAST(value AS VARCHAR) AS value FROM ({sampled_sql}) WHERE value IS NOT NULL{null_string_filter} ORDER BY value LIMIT {}",
11790 threshold + 1
11791 );
11792 let mut statement = connection
11793 .prepare(&sql)
11794 .map_err(|error| infrastructure_error("prepare dataset metadata profiling query", error))?;
11795 let rows = statement
11796 .query_map([], |row| row.get::<_, String>(0))
11797 .map_err(|error| infrastructure_error("profile dataset metadata values", error))?;
11798 let mut values = Vec::new();
11799 for row in rows {
11800 values.push(
11801 row.map_err(|error| infrastructure_error("read dataset metadata profile row", error))?,
11802 );
11803 }
11804 if values.len() > threshold {
11805 Ok(Vec::new())
11806 } else {
11807 Ok(values)
11808 }
11809}
11810
11811fn quote_sql_identifier(value: &str) -> String {
11812 format!("\"{}\"", value.replace('"', "\"\""))
11813}
11814
11815fn sql_string_literal(value: &str) -> String {
11816 format!("'{}'", value.replace('\'', "''"))
11817}
11818
11819fn metadata_value_type_id(
11820 proposal: &FieldMetadataProposal,
11821) -> Result<Option<ahri_tre_types::ValueTypeId>, AppError> {
11822 if let Some(value_type_id) = proposal.value_type_id {
11823 let value_type_id = ahri_tre_types::ValueTypeId(value_type_id);
11824 ahri_tre_types::TreValueType::from_value_type_id(value_type_id)
11825 .map_err(|error| AppError::Validation(error.to_string()))?;
11826 return Ok(Some(value_type_id));
11827 }
11828
11829 proposal
11830 .value_type
11831 .as_deref()
11832 .map(value_type_id_for_metadata_name)
11833 .transpose()
11834}
11835
11836fn compare_optional_enrichment(
11837 warnings: &mut Vec<String>,
11838 strict_metadata: bool,
11839 prefix: &str,
11840 name: &str,
11841 field: &str,
11842 existing: Option<&str>,
11843 proposed: Option<&str>,
11844) -> Result<(), AppError> {
11845 let Some(proposed) = proposed else {
11846 return Ok(());
11847 };
11848 if existing == Some(proposed) {
11849 return Ok(());
11850 }
11851 let message = match existing {
11852 Some(existing) => {
11853 format!(
11854 "{prefix}: {name} existing {field} {existing:?} differs from proposed {proposed:?}"
11855 )
11856 }
11857 None => format!("{prefix}: {name} existing {field} is absent but proposed {proposed:?}"),
11858 };
11859 if strict_metadata {
11860 Err(AppError::Validation(message))
11861 } else {
11862 warnings.push(message);
11863 Ok(())
11864 }
11865}
11866
11867fn value_type_id_for_metadata_name(
11868 value_type: &str,
11869) -> Result<ahri_tre_types::ValueTypeId, AppError> {
11870 let lowered = value_type.trim().to_ascii_lowercase();
11871 let value_type = match lowered.as_str() {
11872 "integer" | "int" | "xsd:integer" | "xsd:int" | "xsd:long" => {
11873 ahri_tre_types::TreValueType::Integer
11874 }
11875 "decimal" | "float" | "double" | "xsd:decimal" | "xsd:float" | "xsd:double" => {
11876 ahri_tre_types::TreValueType::Float
11877 }
11878 "string" | "text" | "xsd:string" => ahri_tre_types::TreValueType::String,
11879 "date" | "xsd:date" => ahri_tre_types::TreValueType::Date,
11880 "datetime" | "timestamp" | "xsd:datetime" => ahri_tre_types::TreValueType::DateTime,
11881 "time" | "xsd:time" => ahri_tre_types::TreValueType::Time,
11882 "enumeration" | "enum" | "categorical" => ahri_tre_types::TreValueType::Enumeration,
11883 "multiresponse" => ahri_tre_types::TreValueType::Multiresponse,
11884 _ => {
11885 return Err(AppError::Validation(format!(
11886 "unsupported TRE value_type metadata {value_type}"
11887 )));
11888 }
11889 };
11890 Ok(value_type.value_type_id())
11891}
11892
11893fn is_enumeration_value_type(value_type: &str) -> bool {
11894 matches!(
11895 value_type_id_for_metadata_name(value_type),
11896 Ok(value_type_id) if value_type_id == ahri_tre_types::TreValueType::Enumeration.value_type_id()
11897 )
11898}
11899
11900fn integer_to_enumeration_value_type_names(existing: &str, incoming: &str) -> bool {
11901 existing.eq_ignore_ascii_case("xsd:integer") && incoming.eq_ignore_ascii_case("enumeration")
11902}
11903
11904fn vocabulary_name_for_variable(variable_name: &str) -> String {
11905 format!("{variable_name}_vocabulary")
11906}
11907
11908fn sanitized_automatic_variable_name(
11909 source_name: &str,
11910 used_names: &mut std::collections::BTreeSet<String>,
11911) -> Result<ahri_tre_types::NcName, AppError> {
11912 let mut candidate = String::new();
11913 for ch in source_name.chars() {
11914 if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' || ch == '.' {
11915 candidate.push(ch);
11916 } else {
11917 candidate.push('_');
11918 }
11919 }
11920 while candidate.contains("__") {
11921 candidate = candidate.replace("__", "_");
11922 }
11923 let candidate = candidate.trim_matches(['_', '-', '.']).to_ascii_lowercase();
11924 let base = if candidate.is_empty() {
11925 "column".to_string()
11926 } else if candidate
11927 .chars()
11928 .next()
11929 .is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_')
11930 {
11931 candidate
11932 } else {
11933 format!("column_{candidate}")
11934 };
11935
11936 let mut name = base.clone();
11937 let mut suffix = 2;
11938 while used_names.contains(&name.to_ascii_lowercase()) {
11939 name = format!("{base}_{suffix}");
11940 suffix += 1;
11941 }
11942 used_names.insert(name.to_ascii_lowercase());
11943
11944 ahri_tre_types::NcName::parse(name).map_err(|error| {
11945 AppError::Validation(format!(
11946 "could not sanitize source column {source_name} into a valid variable name: {error}"
11947 ))
11948 })
11949}
11950
11951fn dataset_variable_arrow_metadata(variable: DatasetVariableMetadata) -> AhriTreFieldMetadata {
11952 let mut metadata = BTreeMap::new();
11953 metadata.insert(
11954 metadata_keys::VARIABLE_ID.to_string(),
11955 variable.variable.variable_id.0.to_string(),
11956 );
11957 metadata.insert(
11958 metadata_keys::VARIABLE_NAME.to_string(),
11959 variable.variable.name.as_str().to_string(),
11960 );
11961 metadata.insert(
11962 metadata_keys::VALUE_TYPE_ID.to_string(),
11963 variable.value_type.value_type_id.0.to_string(),
11964 );
11965 metadata.insert(
11966 metadata_keys::VALUE_TYPE.to_string(),
11967 variable.value_type.value_type,
11968 );
11969 metadata.insert(
11970 metadata_keys::ROW_ROLE.to_string(),
11971 key_role_metadata_value(variable.link.row_role).to_string(),
11972 );
11973 insert_optional_metadata(
11974 &mut metadata,
11975 metadata_keys::VALUE_FORMAT,
11976 variable.variable.value_format.as_deref(),
11977 );
11978 insert_optional_metadata(
11979 &mut metadata,
11980 metadata_keys::DESCRIPTION,
11981 variable.variable.description.as_deref(),
11982 );
11983 insert_optional_metadata(
11984 &mut metadata,
11985 metadata_keys::NOTE,
11986 variable.variable.note.as_deref(),
11987 );
11988 insert_optional_metadata(
11989 &mut metadata,
11990 metadata_keys::ONTOLOGY_NAMESPACE,
11991 variable.variable.ontology_namespace.as_deref(),
11992 );
11993 insert_optional_metadata(
11994 &mut metadata,
11995 metadata_keys::ONTOLOGY_CLASS,
11996 variable.variable.ontology_class.as_deref(),
11997 );
11998 if let Some(vocabulary) = variable.vocabulary {
11999 metadata.insert(
12000 metadata_keys::VOCABULARY_ID.to_string(),
12001 vocabulary.vocabulary_id.0.to_string(),
12002 );
12003 metadata.insert(
12004 metadata_keys::VOCABULARY_NAME.to_string(),
12005 vocabulary.name.as_str().to_string(),
12006 );
12007 }
12008
12009 AhriTreFieldMetadata {
12010 field_name: variable.variable.name.as_str().to_string(),
12011 metadata,
12012 }
12013}
12014
12015fn insert_optional_metadata(
12016 metadata: &mut BTreeMap<String, String>,
12017 key: &str,
12018 value: Option<&str>,
12019) {
12020 if let Some(value) = value
12021 && !value.is_empty()
12022 {
12023 metadata.insert(key.to_string(), value.to_string());
12024 }
12025}
12026
12027fn key_role_metadata_value(key_role: ahri_tre_types::KeyRole) -> &'static str {
12028 match key_role {
12029 ahri_tre_types::KeyRole::None => "none",
12030 ahri_tre_types::KeyRole::Record => "record",
12031 ahri_tre_types::KeyRole::External => "external",
12032 }
12033}
12034
12035fn default_value_type_records() -> Vec<ahri_tre_types::ValueTypeRecord> {
12036 ahri_tre_types::TreValueType::SUPPORTED
12037 .into_iter()
12038 .map(|value_type| ahri_tre_types::ValueTypeRecord {
12039 value_type_id: value_type.value_type_id(),
12040 value_type: value_type.datastore_name().to_string(),
12041 description: Some(value_type_description(value_type).to_string()),
12042 })
12043 .collect()
12044}
12045
12046fn value_type_description(value_type: ahri_tre_types::TreValueType) -> &'static str {
12047 match value_type {
12048 ahri_tre_types::TreValueType::Integer => "Integer values",
12049 ahri_tre_types::TreValueType::Float => "Floating point values",
12050 ahri_tre_types::TreValueType::String => "String values",
12051 ahri_tre_types::TreValueType::Date => "Date values",
12052 ahri_tre_types::TreValueType::DateTime => "Date-time values",
12053 ahri_tre_types::TreValueType::Time => "Time values",
12054 ahri_tre_types::TreValueType::Enumeration => "Categorical values",
12055 ahri_tre_types::TreValueType::Multiresponse => "Multiple categorical responses",
12056 }
12057}
12058
12059fn validate_metadata_key_role(
12060 _field: &'static str,
12061 value: ahri_tre_types::KeyRole,
12062) -> Result<(), AppError> {
12063 match value {
12064 ahri_tre_types::KeyRole::None
12065 | ahri_tre_types::KeyRole::Record
12066 | ahri_tre_types::KeyRole::External => Ok(()),
12067 }
12068}
12069
12070fn infrastructure_error(context: &str, error: impl std::fmt::Display) -> AppError {
12071 AppError::Infrastructure(format!("{context}: {error}"))
12072}
12073
12074async fn run_blocking_lake_work<T>(
12075 operation: impl FnOnce() -> Result<T, ahri_tre_lake::LakeError> + Send + 'static,
12076) -> Result<T, ahri_tre_lake::LakeError>
12077where
12078 T: Send + 'static,
12079{
12080 if tokio::runtime::Handle::try_current().is_ok() {
12081 tokio::task::spawn_blocking(operation).await.map_err(|_| {
12082 ahri_tre_lake::LakeError::Io(std::io::Error::other("Lake worker stopped"))
12083 })?
12084 } else {
12085 let (sender, receiver) = tokio::sync::oneshot::channel();
12086 std::thread::Builder::new()
12087 .name("ahri-tre-lake-worker".to_string())
12088 .spawn(move || {
12089 let _ = sender.send(operation());
12090 })
12091 .map_err(ahri_tre_lake::LakeError::Io)?;
12092 receiver.await.map_err(|_| {
12093 ahri_tre_lake::LakeError::Io(std::io::Error::other("Lake worker stopped"))
12094 })?
12095 }
12096}
12097
12098async fn run_blocking_app_work<T>(
12099 operation: impl FnOnce() -> Result<T, AppError> + Send + 'static,
12100) -> Result<T, AppError>
12101where
12102 T: Send + 'static,
12103{
12104 if tokio::runtime::Handle::try_current().is_ok() {
12105 tokio::task::spawn_blocking(operation)
12106 .await
12107 .map_err(|_| AppError::Infrastructure("blocking worker stopped".to_string()))?
12108 } else {
12109 let (sender, receiver) = tokio::sync::oneshot::channel();
12110 std::thread::Builder::new()
12111 .name("ahri-tre-app-worker".to_string())
12112 .spawn(move || {
12113 let _ = sender.send(operation());
12114 })
12115 .map_err(|error| AppError::Infrastructure(format!("start blocking worker: {error}")))?;
12116 receiver
12117 .await
12118 .map_err(|_| AppError::Infrastructure("blocking worker stopped".to_string()))?
12119 }
12120}
12121
12122async fn run_cancellation_safe_app_work<T>(
12123 operation: impl std::future::Future<Output = Result<T, AppError>> + Send + 'static,
12124) -> Result<T, AppError>
12125where
12126 T: Send + 'static,
12127{
12128 let (sender, receiver) = tokio::sync::oneshot::channel();
12129 std::thread::Builder::new()
12130 .name("ahri-tre-ingest-metadata".to_string())
12131 .spawn(move || {
12132 let result = tokio::runtime::Builder::new_current_thread()
12133 .enable_all()
12134 .build()
12135 .map_err(|error| {
12136 AppError::Infrastructure(format!("start ingest metadata runtime: {error}"))
12137 })
12138 .and_then(|runtime| runtime.block_on(operation));
12139 let _ = sender.send(result);
12140 })
12141 .map_err(|error| {
12142 AppError::Infrastructure(format!("start ingest metadata worker: {error}"))
12143 })?;
12144 receiver
12145 .await
12146 .map_err(|_| AppError::Infrastructure("ingest metadata worker stopped".to_string()))?
12147}
12148
12149fn redcap_error(context: &str, error: ahri_tre_redcap::RedcapError) -> AppError {
12150 match error {
12151 ahri_tre_redcap::RedcapError::InvalidConfig(_)
12152 | ahri_tre_redcap::RedcapError::InvalidMetadata(_)
12153 | ahri_tre_redcap::RedcapError::Encoding(_) => {
12154 AppError::Validation(format!("{context}: {error}"))
12155 }
12156 ahri_tre_redcap::RedcapError::HttpStatus { .. }
12157 | ahri_tre_redcap::RedcapError::Transport { .. }
12158 | ahri_tre_redcap::RedcapError::Http(_)
12159 | ahri_tre_redcap::RedcapError::Io(_)
12160 | ahri_tre_redcap::RedcapError::Json(_) => {
12161 AppError::Infrastructure(format!("{context}: {error}"))
12162 }
12163 }
12164}
12165
12166fn apply_redcap_skip_forms(forms: &mut Vec<RedcapForm>, skip_forms: &[String]) {
12167 if skip_forms.is_empty() {
12168 return;
12169 }
12170 let normalized_skip_forms = skip_forms
12171 .iter()
12172 .map(|form| form.trim())
12173 .filter(|form| !form.is_empty())
12174 .collect::<Vec<_>>();
12175 forms.retain(|form| {
12176 !normalized_skip_forms
12177 .iter()
12178 .any(|skip_form| form.name == *skip_form)
12179 });
12180}
12181
12182fn redcap_empty_form_selection_message(skip_forms: &[String]) -> String {
12183 if skip_forms.is_empty() {
12184 "REDCap metadata did not contain any ingestible forms".to_string()
12185 } else {
12186 "REDCap form selection is empty after applying --skip-form filters".to_string()
12187 }
12188}
12189
12190#[derive(Debug)]
12191struct AcquiredIngestFileSource {
12192 path: PathBuf,
12193}
12194
12195impl Drop for AcquiredIngestFileSource {
12196 fn drop(&mut self) {
12197 if temporary_https_ingest_path(&self.path) {
12198 let _ = fs::remove_file(&self.path);
12199 }
12200 }
12201}
12202
12203pub fn acquire_ingest_file_content(
12205 source: &IngestFileSource,
12206 asset_name: &str,
12207) -> Result<IngestFileContent, AppError> {
12208 let acquired = acquire_ingest_file_source(source, asset_name)?;
12209 let source_file_name = acquired
12210 .path
12211 .file_name()
12212 .and_then(|name| name.to_str())
12213 .ok_or_else(|| AppError::Validation("ingest source filename is invalid".to_string()))?
12214 .to_string();
12215 let file = fs::File::open(&acquired.path).map_err(|error| {
12216 AppError::Infrastructure(format!("open acquired ingest content: {error}"))
12217 })?;
12218 let content_length = file.metadata().ok().map(|metadata| metadata.len());
12219 Ok(IngestFileContent::new(
12220 source_file_name,
12221 None,
12222 content_length,
12223 None,
12224 file,
12225 ))
12226}
12227
12228fn acquire_file_workflow_content(
12229 source: &IngestFileSource,
12230 asset_name: &str,
12231 requested_edam_format: Option<&str>,
12232 lake_root: &str,
12233 validate_without_session: bool,
12234) -> Result<(IngestFileContent, String), AppError> {
12235 let acquired = acquire_ingest_file_source(source, asset_name)?;
12236 let edam_format = resolve_ingest_edam_format(&acquired.path, requested_edam_format)?;
12237 if validate_without_session && edam_format.eq_ignore_ascii_case("EDAM:format_3752") {
12238 let connection = duckdb::Connection::open_in_memory()
12239 .map_err(|error| infrastructure_error("open ingest validation database", error))?;
12240 DuckLakeAdapter::new(lake_root)
12241 .validate_dataset_file(
12242 &connection,
12243 &acquired.path,
12244 DatasetFileFormat::Csv,
12245 &DatasetFileReadOptions::default(),
12246 )
12247 .map_err(|error| match error {
12248 ahri_tre_lake::LakeError::DuckDb(_) => {
12249 AppError::Validation("Datafile content is invalid".to_string())
12250 }
12251 other => infrastructure_error("validate ingest datafile", other),
12252 })?;
12253 }
12254 let source_file_name = acquired
12255 .path
12256 .file_name()
12257 .and_then(|name| name.to_str())
12258 .ok_or_else(|| AppError::Validation("ingest source filename is invalid".to_string()))?
12259 .to_string();
12260 let file = fs::File::open(&acquired.path).map_err(|error| {
12261 AppError::Infrastructure(format!("open ingest source content: {error}"))
12262 })?;
12263 let content_length = file.metadata().ok().map(|metadata| metadata.len());
12264 Ok((
12265 IngestFileContent::new(source_file_name, None, content_length, None, file),
12266 edam_format,
12267 ))
12268}
12269
12270fn acquire_ingest_file_source(
12271 source: &IngestFileSource,
12272 asset_name: &str,
12273) -> Result<AcquiredIngestFileSource, AppError> {
12274 match source {
12275 IngestFileSource::LocalPath { path } => {
12276 let path = PathBuf::from(path);
12277 if !path.is_file() {
12278 return Err(AppError::Validation(format!(
12279 "ingest source file not found: {}",
12280 path.display()
12281 )));
12282 }
12283 Ok(AcquiredIngestFileSource { path })
12284 }
12285 IngestFileSource::HttpsUri { uri } => acquire_https_ingest_file_source(uri, asset_name),
12286 }
12287}
12288
12289fn acquire_https_ingest_file_source(
12290 uri: &str,
12291 asset_name: &str,
12292) -> Result<AcquiredIngestFileSource, AppError> {
12293 let url = parse_supported_https_ingest_url(uri)?;
12294 let redacted_uri = redact_uri(&url);
12295 let response = https_ingest_client()?
12296 .get(url.clone())
12297 .send()
12298 .map_err(|error| {
12299 AppError::Infrastructure(format!(
12300 "fetch HTTPS ingest source {redacted_uri}: {}",
12301 redact_reqwest_error(error)
12302 ))
12303 })?;
12304 let status = response.status();
12305 if !status.is_success() {
12306 return Err(AppError::Infrastructure(format!(
12307 "fetch HTTPS ingest source {redacted_uri}: HTTP status {}",
12308 status.as_u16()
12309 )));
12310 }
12311 let max_bytes = https_ingest_max_bytes();
12312 if let Some(length) = response.content_length()
12313 && length > max_bytes
12314 {
12315 return Err(AppError::Validation(format!(
12316 "HTTPS ingest source {redacted_uri} exceeds the {} byte response limit",
12317 max_bytes
12318 )));
12319 }
12320
12321 let source_file_name = https_source_file_name(&url, asset_name)?;
12322 let destination = https_ingest_temp_path(&source_file_name);
12323 let mut reader = response;
12324 let mut writer = fs::File::create(&destination).map_err(|error| {
12325 AppError::Infrastructure(format!("create HTTPS ingest staging file: {error}"))
12326 })?;
12327 let copied = copy_bounded(&mut reader, &mut writer, max_bytes).inspect_err(|_error| {
12328 let _ = fs::remove_file(&destination);
12329 })?;
12330 if copied == 0 {
12331 let _ = fs::remove_file(&destination);
12332 return Err(AppError::Validation(format!(
12333 "HTTPS ingest source {redacted_uri} returned empty content"
12334 )));
12335 }
12336 Ok(AcquiredIngestFileSource { path: destination })
12337}
12338
12339fn parse_supported_https_ingest_url(uri: &str) -> Result<Url, AppError> {
12340 let url = Url::parse(uri)
12341 .map_err(|error| AppError::Validation(format!("ingest source URI is invalid: {error}")))?;
12342 if url.scheme() != "https" {
12343 return Err(AppError::Validation(format!(
12344 "unsupported ingest source URI scheme: {}",
12345 url.scheme()
12346 )));
12347 }
12348 Ok(url)
12349}
12350
12351fn https_ingest_client() -> Result<Client, AppError> {
12352 let builder = Client::builder()
12353 .timeout(https_ingest_timeout())
12354 .redirect(Policy::custom(https_ingest_redirect_policy));
12355 #[cfg(test)]
12356 let builder = {
12357 if let Some(cert) = test_https_root_certificate()? {
12358 builder.add_root_certificate(cert)
12359 } else {
12360 builder
12361 }
12362 };
12363 builder.build().map_err(|error| {
12364 AppError::Infrastructure(format!("configure HTTPS ingest client: {error}"))
12365 })
12366}
12367
12368#[cfg(test)]
12369fn test_https_root_certificate() -> Result<Option<reqwest::Certificate>, AppError> {
12370 let certificate = TEST_HTTPS_ROOT_CERTIFICATE_DER
12371 .lock()
12372 .expect("test HTTPS root certificate lock should not be poisoned")
12373 .clone();
12374 certificate
12375 .map(|certificate| {
12376 reqwest::Certificate::from_der(&certificate).map_err(|error| {
12377 AppError::Infrastructure(format!("configure test HTTPS root certificate: {error}"))
12378 })
12379 })
12380 .transpose()
12381}
12382
12383fn https_ingest_max_bytes() -> u64 {
12384 #[cfg(test)]
12385 {
12386 TEST_HTTPS_MAX_BYTES
12387 .lock()
12388 .expect("test HTTPS max bytes lock should not be poisoned")
12389 .unwrap_or(HTTPS_INGEST_MAX_BYTES)
12390 }
12391 #[cfg(not(test))]
12392 {
12393 HTTPS_INGEST_MAX_BYTES
12394 }
12395}
12396
12397fn https_ingest_timeout() -> StdDuration {
12398 #[cfg(test)]
12399 {
12400 TEST_HTTPS_TIMEOUT
12401 .lock()
12402 .expect("test HTTPS timeout lock should not be poisoned")
12403 .unwrap_or(HTTPS_INGEST_TIMEOUT)
12404 }
12405 #[cfg(not(test))]
12406 {
12407 HTTPS_INGEST_TIMEOUT
12408 }
12409}
12410
12411fn https_ingest_redirect_policy(attempt: Attempt<'_>) -> reqwest::redirect::Action {
12412 match validate_https_ingest_redirect(attempt.previous().len(), attempt.url()) {
12413 Ok(()) => attempt.follow(),
12414 Err(message) => attempt.error(message),
12415 }
12416}
12417
12418fn validate_https_ingest_redirect(previous_count: usize, url: &Url) -> Result<(), &'static str> {
12419 if previous_count >= HTTPS_INGEST_MAX_REDIRECTS {
12420 return Err("HTTPS ingest redirect limit exceeded");
12421 }
12422 if url.scheme() != "https" {
12423 return Err("HTTPS ingest redirect target uses an unsupported scheme");
12424 }
12425 Ok(())
12426}
12427
12428fn copy_bounded<R: Read, W: std::io::Write>(
12429 reader: &mut R,
12430 writer: &mut W,
12431 max_bytes: u64,
12432) -> Result<u64, AppError> {
12433 let mut total = 0_u64;
12434 let mut buffer = [0_u8; 16 * 1024];
12435 loop {
12436 let read = reader.read(&mut buffer).map_err(|error| {
12437 AppError::Infrastructure(format!("read HTTPS ingest source: {error}"))
12438 })?;
12439 if read == 0 {
12440 return Ok(total);
12441 }
12442 total = total.saturating_add(read as u64);
12443 if total > max_bytes {
12444 return Err(AppError::Validation(format!(
12445 "HTTPS ingest source exceeds the {} byte response limit",
12446 max_bytes
12447 )));
12448 }
12449 writer.write_all(&buffer[..read]).map_err(|error| {
12450 AppError::Infrastructure(format!("write HTTPS ingest staging file: {error}"))
12451 })?;
12452 }
12453}
12454
12455fn redact_reqwest_error(error: reqwest::Error) -> String {
12456 error.without_url().to_string()
12457}
12458
12459fn redact_uri(url: &Url) -> String {
12460 ahri_tre_security::redact_uri(url.as_str(), UriRedactionMode::ExternalSourceLineage)
12461}
12462
12463fn https_source_file_name(url: &Url, asset_name: &str) -> Result<String, AppError> {
12464 let candidate = url
12465 .path_segments()
12466 .and_then(|mut segments| segments.next_back())
12467 .filter(|segment| !segment.trim().is_empty())
12468 .unwrap_or(asset_name);
12469 if candidate.contains('/') || candidate.contains('\\') {
12470 return Err(AppError::Validation(
12471 "HTTPS ingest source filename is invalid".to_string(),
12472 ));
12473 }
12474 Ok(candidate.to_string())
12475}
12476
12477fn https_ingest_temp_path(source_file_name: &str) -> PathBuf {
12478 env::temp_dir().join(format!(
12479 "ahri-tre-https-ingest-{}-{source_file_name}",
12480 uuid::Uuid::new_v4()
12481 ))
12482}
12483
12484fn temporary_https_ingest_path(path: &Path) -> bool {
12485 path.file_name()
12486 .and_then(|name| name.to_str())
12487 .is_some_and(|name| name.starts_with("ahri-tre-https-ingest-"))
12488}
12489
12490fn enrich_transformation_source(
12491 transformation: &ahri_tre_types::NewTransformationRecord,
12492) -> ahri_tre_types::NewTransformationRecord {
12493 enrich_transformation_source_with_git(transformation, None, run_git)
12494}
12495
12496fn enrich_transformation_source_with_git(
12497 transformation: &ahri_tre_types::NewTransformationRecord,
12498 caller_file: Option<&str>,
12499 run_git: impl Fn(&[&str]) -> Option<String>,
12500) -> ahri_tre_types::NewTransformationRecord {
12501 if transformation.repository_url.is_some()
12502 && transformation.commit_hash.is_some()
12503 && transformation.file_path.is_some()
12504 {
12505 return transformation.clone();
12506 }
12507
12508 let mut enriched = transformation.clone();
12509 let source_file = transformation.file_path.as_deref().or(caller_file);
12510 let repo_root = source_context_dirs(source_file)
12511 .into_iter()
12512 .find_map(|source_context| {
12513 let source_context = path_argument(&source_context);
12514 run_git(&[
12515 "-C",
12516 source_context.as_str(),
12517 "rev-parse",
12518 "--show-toplevel",
12519 ])
12520 });
12521 let Some(repo_root) = repo_root else {
12522 return enriched;
12523 };
12524
12525 if enriched.file_path.is_none()
12526 && let Some(source_file) =
12527 source_file.and_then(|path| source_file_relative_to_repo(path, &repo_root))
12528 {
12529 enriched.file_path = Some(source_file);
12530 }
12531
12532 if enriched.repository_url.is_none() {
12533 enriched.repository_url = run_git(&[
12534 "-C",
12535 repo_root.as_str(),
12536 "config",
12537 "--get",
12538 "remote.origin.url",
12539 ])
12540 .map(|url| normalize_remote_url(&url))
12541 .filter(|url| !url.is_empty());
12542 }
12543
12544 if enriched.commit_hash.is_none() {
12545 enriched.commit_hash = run_git(&["-C", repo_root.as_str(), "rev-parse", "HEAD"])
12546 .map(|hash| short_commit_hash(&hash).to_string());
12547 }
12548
12549 enriched
12550}
12551
12552fn source_file_relative_to_repo(source_file: &str, repo_root: &str) -> Option<String> {
12553 let repo_root = Path::new(repo_root);
12554 source_file_candidates(source_file)
12555 .into_iter()
12556 .find_map(|candidate| {
12557 candidate
12558 .strip_prefix(repo_root)
12559 .ok()
12560 .map(normalized_path_string)
12561 })
12562}
12563
12564fn source_file_candidates(source_file: &str) -> Vec<PathBuf> {
12565 let path = Path::new(source_file);
12566 if path.is_absolute() {
12567 return vec![path.to_path_buf()];
12568 }
12569
12570 let mut candidates = Vec::new();
12571 if let Ok(current_dir) = env::current_dir() {
12572 push_unique_path(&mut candidates, current_dir.join(path));
12573 }
12574
12575 let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
12576 for ancestor in manifest_dir.ancestors() {
12577 push_unique_path(&mut candidates, ancestor.join(path));
12578 }
12579
12580 candidates
12581}
12582
12583fn source_context_dirs(file_path: Option<&str>) -> Vec<PathBuf> {
12584 let mut candidates = Vec::new();
12585
12586 if let Some(file_path) = file_path {
12587 let path = Path::new(file_path);
12588 if path.is_absolute() {
12589 push_source_parent(&mut candidates, path.to_path_buf());
12590 } else {
12591 if let Ok(current_dir) = env::current_dir() {
12592 push_source_parent(&mut candidates, current_dir.join(path));
12593 }
12594
12595 let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
12596 for ancestor in manifest_dir.ancestors() {
12597 push_source_parent(&mut candidates, ancestor.join(path));
12598 }
12599 }
12600 }
12601
12602 if let Ok(current_dir) = env::current_dir() {
12603 push_unique_path(&mut candidates, current_dir);
12604 }
12605
12606 candidates
12607}
12608
12609fn push_source_parent(candidates: &mut Vec<PathBuf>, source_path: PathBuf) {
12610 if let Some(parent) = source_path.parent() {
12611 push_unique_path(candidates, parent.to_path_buf());
12612 }
12613}
12614
12615fn push_unique_path(candidates: &mut Vec<PathBuf>, path: PathBuf) {
12616 if !candidates.iter().any(|existing| existing == &path) {
12617 candidates.push(path);
12618 }
12619}
12620
12621fn run_git(args: &[&str]) -> Option<String> {
12622 let output = Command::new("git").args(args).output().ok()?;
12623 if !output.status.success() {
12624 return None;
12625 }
12626
12627 let text = String::from_utf8(output.stdout).ok()?;
12628 let trimmed = text.trim();
12629 (!trimmed.is_empty()).then(|| trimmed.to_string())
12630}
12631
12632fn path_argument(path: &Path) -> String {
12633 path.to_string_lossy().into_owned()
12634}
12635
12636fn normalized_path_string(path: &Path) -> String {
12637 path.to_string_lossy().replace('\\', "/")
12638}
12639
12640fn normalize_remote_url(url: &str) -> String {
12641 let trimmed = url.trim();
12642 if let Some(rest) = trimmed.strip_prefix("git@")
12643 && let Some((host, path)) = rest.split_once(':')
12644 {
12645 return trim_git_suffix(&format!("https://{host}/{path}"));
12646 }
12647
12648 trim_git_suffix(trimmed)
12649}
12650
12651fn trim_git_suffix(value: &str) -> String {
12652 value.strip_suffix(".git").unwrap_or(value).to_string()
12653}
12654
12655fn short_commit_hash(hash: &str) -> &str {
12656 hash.get(..7).unwrap_or(hash)
12657}
12658
12659fn core_error(context: &str, error: CoreError) -> AppError {
12660 match error {
12661 CoreError::Validation(_) => AppError::Validation(format!("{context}: {error}")),
12662 CoreError::NotFound(_) => AppError::NotFound(format!("{context}: {error}")),
12663 CoreError::Conflict(_) => AppError::Conflict(format!("{context}: {error}")),
12664 CoreError::NotImplemented(_) | CoreError::Infrastructure(_) => {
12665 AppError::Infrastructure(format!("{context}: {error}"))
12666 }
12667 }
12668}
12669
12670#[derive(Default)]
12671struct CountAccumulator {
12672 count: u64,
12673 unsupported: bool,
12674 unavailable: bool,
12675}
12676
12677impl CountAccumulator {
12678 fn add(&mut self, count: usize) {
12679 self.count += count as u64;
12680 }
12681
12682 fn record_error(&mut self, error: CoreError) {
12683 match error {
12684 CoreError::NotImplemented(_) => self.unsupported = true,
12685 _ => self.unavailable = true,
12686 }
12687 }
12688
12689 fn record_unavailable(&mut self) {
12690 self.unavailable = true;
12691 }
12692
12693 fn into_count(self) -> DatastoreContentCount {
12694 if self.unsupported {
12695 DatastoreContentCount::unsupported()
12696 } else if self.unavailable {
12697 DatastoreContentCount::unavailable()
12698 } else {
12699 DatastoreContentCount::available(self.count)
12700 }
12701 }
12702}
12703
12704fn governance_core_error(context: &str, error: CoreError) -> AppError {
12705 match error {
12706 CoreError::Infrastructure(message) if looks_like_authorization_denial(&message) => {
12707 AppError::Validation(format!("{context}: authorization denied"))
12708 }
12709 other => core_error(context, other),
12710 }
12711}
12712
12713fn looks_like_authorization_denial(message: &str) -> bool {
12714 let lowered = message.to_ascii_lowercase();
12715 lowered.contains("permission denied")
12716 || lowered.contains("row-level security")
12717 || lowered.contains("42501")
12718 || lowered.contains("authorization denied")
12719}
12720
12721fn parse_domain_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12722 ahri_tre_types::NcName::parse(name)
12723 .map_err(|error| AppError::Validation(format!("domain name is invalid: {error}")))
12724}
12725
12726fn domain_selector_label(selector: &DomainSelector) -> String {
12727 match selector {
12728 DomainSelector::Id { domain_id } => domain_id.0.to_string(),
12729 DomainSelector::Name { name } => name.clone(),
12730 }
12731}
12732
12733fn app_study_search_query(request: SearchStudiesRequest) -> Result<StudySearchQuery, AppError> {
12734 Ok(StudySearchQuery {
12735 kind: request.kind,
12736 predicates: request.predicates,
12737 limit: request.limit,
12738 after: request.after,
12739 })
12740}
12741
12742fn app_dataset_search_query(
12743 request: SearchDatasetsRequest,
12744) -> Result<DatasetSearchQuery, AppError> {
12745 Ok(DatasetSearchQuery {
12746 kind: request.kind,
12747 predicates: request.predicates,
12748 limit: request.limit,
12749 after: request.after,
12750 })
12751}
12752
12753fn app_datafile_search_query(
12754 request: SearchDataFilesRequest,
12755) -> Result<DataFileSearchQuery, AppError> {
12756 Ok(DataFileSearchQuery {
12757 kind: request.kind,
12758 predicates: request.predicates,
12759 limit: request.limit,
12760 after: request.after,
12761 })
12762}
12763
12764fn app_variable_search_query(
12765 request: SearchVariablesRequest,
12766) -> Result<VariableSearchQuery, AppError> {
12767 Ok(VariableSearchQuery {
12768 kind: request.kind,
12769 predicates: request.predicates,
12770 limit: request.limit,
12771 after: request.after,
12772 })
12773}
12774
12775fn app_entity_search_query(request: SearchEntitiesRequest) -> Result<EntitySearchQuery, AppError> {
12776 Ok(EntitySearchQuery {
12777 kind: request.kind,
12778 predicates: request.predicates,
12779 limit: request.limit,
12780 after: request.after,
12781 })
12782}
12783
12784fn app_relation_search_query(
12785 request: SearchRelationsRequest,
12786) -> Result<RelationSearchQuery, AppError> {
12787 Ok(RelationSearchQuery {
12788 kind: request.kind,
12789 predicates: request.predicates,
12790 limit: request.limit,
12791 after: request.after,
12792 })
12793}
12794
12795fn study_selector_label(selector: &StudySelector) -> String {
12796 match selector {
12797 StudySelector::Id { study_id } => study_id.0.to_string(),
12798 StudySelector::Name { name, .. } => name.clone(),
12799 }
12800}
12801
12802fn variable_selector_label(selector: &VariableSelector) -> String {
12803 match selector {
12804 VariableSelector::Id { variable_id } => variable_id.0.to_string(),
12805 VariableSelector::Name { name, .. } => name.clone(),
12806 }
12807}
12808
12809fn semantic_delete_validation_error(
12810 code: impl Into<String>,
12811 message: impl Into<String>,
12812) -> SemanticDeleteValidationError {
12813 SemanticDeleteValidationError {
12814 code: code.into(),
12815 message: message.into(),
12816 }
12817}
12818
12819fn semantic_dependency(
12820 kind: impl Into<String>,
12821 identifier: impl ToString,
12822 description: impl Into<String>,
12823) -> SemanticDeleteDependency {
12824 SemanticDeleteDependency {
12825 kind: kind.into(),
12826 identifier: identifier.to_string(),
12827 description: description.into(),
12828 }
12829}
12830
12831fn semantic_identity(
12832 kind: SemanticDeleteTargetKind,
12833 domain_id: ahri_tre_types::DomainId,
12834 id: impl ToString,
12835 domain: Option<String>,
12836 name: &str,
12837) -> SemanticDeleteTargetIdentity {
12838 SemanticDeleteTargetIdentity {
12839 domain_id,
12840 kind,
12841 id: id.to_string(),
12842 domain,
12843 name: name.to_string(),
12844 }
12845}
12846
12847fn finalize_semantic_summary(summary: &mut SemanticDeleteDependencySummary) {
12848 if summary.has_blockers() {
12849 summary.status = SemanticDeleteDependencyStatus::Blocked;
12850 }
12851}
12852
12853fn parse_semantic_i64_id(id: &str) -> i64 {
12854 id.parse::<i64>()
12855 .expect("semantic delete plan id should be generated by the app")
12856}
12857
12858fn parse_study_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12859 ahri_tre_types::NcName::parse(name)
12860 .map_err(|error| AppError::Validation(format!("study name is invalid: {error}")))
12861}
12862
12863fn parse_variable_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12864 ahri_tre_types::NcName::parse(name)
12865 .map_err(|error| AppError::Validation(format!("variable name is invalid: {error}")))
12866}
12867
12868fn parse_vocabulary_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12869 ahri_tre_types::NcName::parse(name)
12870 .map_err(|error| AppError::Validation(format!("vocabulary name is invalid: {error}")))
12871}
12872
12873fn parse_entity_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12874 ahri_tre_types::NcName::parse(name)
12875 .map_err(|error| AppError::Validation(format!("entity name is invalid: {error}")))
12876}
12877
12878fn parse_entity_relation_name(name: String) -> Result<ahri_tre_types::NcName, AppError> {
12879 ahri_tre_types::NcName::parse(name)
12880 .map_err(|error| AppError::Validation(format!("entity relation name is invalid: {error}")))
12881}
12882
12883fn require_positive_transformation_id(
12884 transformation_id: ahri_tre_types::TransformationId,
12885) -> Result<(), AppError> {
12886 if transformation_id.0 <= 0 {
12887 return Err(AppError::Validation(format!(
12888 "transformation-id must be a positive integer: {}",
12889 transformation_id.0
12890 )));
12891 }
12892 Ok(())
12893}
12894
12895fn require_dataset_variable_id(
12896 variables: &[DatasetVariableMetadata],
12897 name: &str,
12898) -> Result<ahri_tre_types::VariableId, AppError> {
12899 let variable_name = parse_variable_name(name.to_string())?;
12900 variables
12901 .iter()
12902 .find(|metadata| metadata.variable.name == variable_name)
12903 .map(|metadata| metadata.variable.variable_id)
12904 .ok_or_else(|| AppError::Validation(dataset_variable_not_found_message(variables, name)))
12905}
12906
12907fn dataset_variable_not_found_message(variables: &[DatasetVariableMetadata], name: &str) -> String {
12908 let available = variables
12909 .iter()
12910 .map(|metadata| metadata.variable.name.as_str())
12911 .collect::<Vec<_>>();
12912 if available.is_empty() {
12913 format!("dataset variable not found: {name}; dataset has no registered variables")
12914 } else {
12915 format!(
12916 "dataset variable not found: {name}; available variables: {}",
12917 available.join(", ")
12918 )
12919 }
12920}
12921
12922struct EntityBatchSourceRow {
12923 row_number: usize,
12924 values: BTreeMap<String, Option<String>>,
12925}
12926
12927struct EntityInstanceDatasetBatchPlan {
12928 external_id: String,
12929 label: Option<String>,
12930}
12931
12932#[derive(Clone, Debug, PartialEq, Eq)]
12933struct RelationInstanceDatasetBatchRawPlan {
12934 external_id: String,
12935 subject_external_id: String,
12936 object_external_id: String,
12937 valid_from: Option<chrono::NaiveDate>,
12938 valid_to: Option<chrono::NaiveDate>,
12939}
12940
12941struct RelationInstanceDatasetBatchPlan {
12942 external_id: String,
12943 subject_entity_instance_id: i64,
12944 object_entity_instance_id: i64,
12945 valid_from: Option<chrono::NaiveDate>,
12946 valid_to: Option<chrono::NaiveDate>,
12947}
12948
12949struct RelationEndpointLookup<'a> {
12950 mapping: Option<ahri_tre_types::StudyEntityInstanceRecord>,
12951 expected_entity_id: ahri_tre_types::EntityId,
12952 relation_external_id: &'a str,
12953 endpoint_role: &'a str,
12954 endpoint_external_id: &'a str,
12955}
12956
12957struct RelationBatchLineage {
12958 transformation_id: Option<ahri_tre_types::TransformationId>,
12959 transformation: Option<ahri_tre_types::TransformationLineageRecord>,
12960}
12961
12962impl RelationBatchLineage {
12963 fn none() -> Self {
12964 Self {
12965 transformation_id: None,
12966 transformation: None,
12967 }
12968 }
12969}
12970
12971fn read_entity_batch_source_rows(
12972 session: &DataStoreSession,
12973 study_id: ahri_tre_types::StudyId,
12974 dataset_name: &str,
12975 version: &ahri_tre_types::AssetVersionRecord,
12976 external_id_variable: &str,
12977 template_columns: &[String],
12978) -> Result<Vec<EntityBatchSourceRow>, AppError> {
12979 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
12980 study_id,
12981 dataset_name,
12982 version.major,
12983 version.minor,
12984 version.patch,
12985 );
12986 let mut columns = vec![external_id_variable.to_string()];
12987 for column in template_columns {
12988 if !columns.iter().any(|existing| existing == column) {
12989 columns.push(column.clone());
12990 }
12991 }
12992 let projections = columns
12993 .iter()
12994 .map(|column| {
12995 let identifier = sql_identifier(column);
12996 format!("CAST({identifier} AS VARCHAR) AS {identifier}")
12997 })
12998 .collect::<Vec<_>>()
12999 .join(", ");
13000 let sql = format!(
13001 "SELECT {projections} FROM {};",
13002 dataset_relation_sql(&relation)
13003 );
13004 let mut statement = session
13005 .lake_connection()
13006 .prepare(&sql)
13007 .map_err(|error| infrastructure_error("prepare entity batch dataset scan", error))?;
13008 let mut rows = statement
13009 .query([])
13010 .map_err(|error| infrastructure_error("scan entity batch dataset", error))?;
13011 let mut row_number = 0;
13012 let mut source_rows = Vec::new();
13013 while let Some(row) = rows
13014 .next()
13015 .map_err(|error| infrastructure_error("read entity batch dataset row", error))?
13016 {
13017 row_number += 1;
13018 let mut values = BTreeMap::new();
13019 for (index, column) in columns.iter().enumerate() {
13020 values.insert(
13021 column.clone(),
13022 row.get::<_, Option<String>>(index).map_err(|error| {
13023 infrastructure_error("decode entity batch dataset row", error)
13024 })?,
13025 );
13026 }
13027 source_rows.push(EntityBatchSourceRow { row_number, values });
13028 }
13029 Ok(source_rows)
13030}
13031
13032fn read_relation_batch_source_rows(
13033 session: &DataStoreSession,
13034 study_id: ahri_tre_types::StudyId,
13035 dataset_name: &str,
13036 version: &ahri_tre_types::AssetVersionRecord,
13037 request: &EnsureRelationInstancesFromDatasetRequest,
13038) -> Result<Vec<EntityBatchSourceRow>, AppError> {
13039 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
13040 study_id,
13041 dataset_name,
13042 version.major,
13043 version.minor,
13044 version.patch,
13045 );
13046 let mut columns = Vec::new();
13047 for column in [
13048 &request.relation_external_id_variable,
13049 &request.subject_external_id_variable,
13050 &request.object_external_id_variable,
13051 ] {
13052 if !columns.iter().any(|existing| existing == column) {
13053 columns.push(column.clone());
13054 }
13055 }
13056 for column in [
13057 request.valid_from_variable.as_ref(),
13058 request.valid_to_variable.as_ref(),
13059 ]
13060 .into_iter()
13061 .flatten()
13062 {
13063 if !columns.iter().any(|existing| existing == column) {
13064 columns.push(column.clone());
13065 }
13066 }
13067 read_dataset_batch_source_rows(session, &relation, &columns, "relation batch dataset")
13068}
13069
13070fn read_dataset_batch_source_rows(
13071 session: &DataStoreSession,
13072 relation: &ahri_tre_lake::DatasetTableRelation,
13073 columns: &[String],
13074 context: &'static str,
13075) -> Result<Vec<EntityBatchSourceRow>, AppError> {
13076 let projections = columns
13077 .iter()
13078 .map(|column| {
13079 let identifier = sql_identifier(column);
13080 format!("CAST({identifier} AS VARCHAR) AS {identifier}")
13081 })
13082 .collect::<Vec<_>>()
13083 .join(", ");
13084 let sql = format!(
13085 "SELECT {projections} FROM {};",
13086 dataset_relation_sql(relation)
13087 );
13088 let mut statement = session
13089 .lake_connection()
13090 .prepare(&sql)
13091 .map_err(|error| infrastructure_error(&format!("prepare {context} scan"), error))?;
13092 let mut rows = statement
13093 .query([])
13094 .map_err(|error| infrastructure_error(&format!("scan {context}"), error))?;
13095 let mut row_number = 0;
13096 let mut source_rows = Vec::new();
13097 while let Some(row) = rows
13098 .next()
13099 .map_err(|error| infrastructure_error(&format!("read {context} row"), error))?
13100 {
13101 row_number += 1;
13102 let mut values = BTreeMap::new();
13103 for (index, column) in columns.iter().enumerate() {
13104 values.insert(
13105 column.clone(),
13106 row.get::<_, Option<String>>(index).map_err(|error| {
13107 infrastructure_error(&format!("decode {context} row"), error)
13108 })?,
13109 );
13110 }
13111 source_rows.push(EntityBatchSourceRow { row_number, values });
13112 }
13113 Ok(source_rows)
13114}
13115
13116fn trimmed_row_value(values: &BTreeMap<String, Option<String>>, column: &str) -> Option<String> {
13117 values
13118 .get(column)
13119 .and_then(|value| value.as_deref())
13120 .map(str::trim)
13121 .filter(|value| !value.is_empty())
13122 .map(str::to_string)
13123}
13124
13125fn parse_relation_batch_date(
13126 values: &BTreeMap<String, Option<String>>,
13127 column: Option<&str>,
13128 row_number: usize,
13129 rejected_row_count: &mut usize,
13130 rejected_rows: &mut Vec<RelationInstanceDatasetBatchRejectedRow>,
13131) -> Result<Option<chrono::NaiveDate>, AppError> {
13132 let Some(column) = column else {
13133 return Ok(None);
13134 };
13135 let Some(value) = trimmed_row_value(values, column) else {
13136 return Ok(None);
13137 };
13138 match chrono::NaiveDate::parse_from_str(&value, "%Y-%m-%d") {
13139 Ok(date) => Ok(Some(date)),
13140 Err(error) => {
13141 *rejected_row_count += 1;
13142 if rejected_rows.len() < 10 {
13143 rejected_rows.push(RelationInstanceDatasetBatchRejectedRow {
13144 row_number,
13145 reason: format!("invalid date in {column}: {error}"),
13146 });
13147 }
13148 Ok(None)
13149 }
13150 }
13151}
13152
13153fn label_template_columns(template: Option<&str>) -> Result<Vec<String>, AppError> {
13154 let Some(template) = template else {
13155 return Ok(Vec::new());
13156 };
13157 let mut columns = Vec::new();
13158 let mut rest = template;
13159 while let Some(open) = rest.find('{') {
13160 if rest[..open].contains('}') {
13161 return Err(AppError::Validation(
13162 "label-template contains unmatched `}`".to_string(),
13163 ));
13164 }
13165 let after_open = &rest[open + 1..];
13166 let close = after_open.find('}').ok_or_else(|| {
13167 AppError::Validation("label-template contains unmatched `{`".to_string())
13168 })?;
13169 let column = &after_open[..close];
13170 if column.is_empty() || column.contains('{') || column.contains('}') {
13171 return Err(AppError::Validation(
13172 "label-template placeholders must be simple {ColumnName} references".to_string(),
13173 ));
13174 }
13175 if !columns.iter().any(|existing| existing == column) {
13176 columns.push(column.to_string());
13177 }
13178 rest = &after_open[close + 1..];
13179 }
13180 if rest.contains('}') {
13181 return Err(AppError::Validation(
13182 "label-template contains unmatched `}`".to_string(),
13183 ));
13184 }
13185 Ok(columns)
13186}
13187
13188fn render_label_template(template: &str, values: &BTreeMap<String, Option<String>>) -> String {
13189 let mut rendered = String::new();
13190 let mut rest = template;
13191 while let Some(open) = rest.find('{') {
13192 rendered.push_str(&rest[..open]);
13193 let after_open = &rest[open + 1..];
13194 let close = after_open
13195 .find('}')
13196 .expect("label template should be validated before rendering");
13197 let column = &after_open[..close];
13198 if let Some(Some(value)) = values.get(column) {
13199 rendered.push_str(value);
13200 }
13201 rest = &after_open[close + 1..];
13202 }
13203 rendered.push_str(rest);
13204 rendered
13205}
13206
13207fn entity_batch_result(
13208 request: &EnsureEntityInstancesFromDatasetRequest,
13209 version: &ahri_tre_types::AssetVersionRecord,
13210 transformation_id: Option<ahri_tre_types::TransformationId>,
13211 transformation: Option<ahri_tre_types::TransformationLineageRecord>,
13212 counts: EntityInstanceDatasetBatchCounts,
13213 conflicts: Vec<EntityInstanceDatasetBatchConflict>,
13214 rejected_rows: Vec<EntityInstanceDatasetBatchRejectedRow>,
13215) -> EntityInstanceDatasetBatchResult {
13216 EntityInstanceDatasetBatchResult {
13217 schema_version: "cli.semantic.entity_batch.output.v1".to_string(),
13218 operation: "entity_instance.ensure_from_dataset".to_string(),
13219 dry_run: request.dry_run,
13220 study: request.study.clone(),
13221 domain: request.domain.clone(),
13222 entity: request.entity.clone(),
13223 dataset: request.dataset.clone(),
13224 requested_version: request.version.clone(),
13225 resolved_version: version_selector_text(version),
13226 external_id_variable: request.external_id_variable.clone(),
13227 label_template: request.label_template.clone(),
13228 transformation_id,
13229 transformation,
13230 counts,
13231 conflicts,
13232 rejected_rows,
13233 }
13234}
13235
13236fn relation_batch_result(
13237 request: &EnsureRelationInstancesFromDatasetRequest,
13238 version: &ahri_tre_types::AssetVersionRecord,
13239 lineage: RelationBatchLineage,
13240 counts: RelationInstanceDatasetBatchCounts,
13241 conflicts: Vec<RelationInstanceDatasetBatchConflict>,
13242 missing_endpoints: Vec<RelationInstanceDatasetBatchMissingEndpoint>,
13243 rejected_rows: Vec<RelationInstanceDatasetBatchRejectedRow>,
13244) -> RelationInstanceDatasetBatchResult {
13245 RelationInstanceDatasetBatchResult {
13246 schema_version: "cli.semantic.relation_batch.output.v1".to_string(),
13247 operation: "entity_relation.instance.ensure_from_dataset".to_string(),
13248 dry_run: request.dry_run,
13249 study: request.study.clone(),
13250 domain: request.domain.clone(),
13251 relation: request.relation.clone(),
13252 dataset: request.dataset.clone(),
13253 requested_version: request.version.clone(),
13254 resolved_version: version_selector_text(version),
13255 relation_external_id_variable: request.relation_external_id_variable.clone(),
13256 subject_external_id_variable: request.subject_external_id_variable.clone(),
13257 object_external_id_variable: request.object_external_id_variable.clone(),
13258 valid_from_variable: request.valid_from_variable.clone(),
13259 valid_to_variable: request.valid_to_variable.clone(),
13260 transformation_id: lineage.transformation_id,
13261 transformation: lineage.transformation,
13262 counts,
13263 conflicts,
13264 missing_endpoints,
13265 rejected_rows,
13266 }
13267}
13268
13269fn version_selector_text(version: &ahri_tre_types::AssetVersionRecord) -> String {
13270 format!("{}.{}.{}", version.major, version.minor, version.patch)
13271}
13272
13273fn dataset_relation_sql(relation: &ahri_tre_lake::DatasetTableRelation) -> String {
13274 format!(
13275 "{}.{}",
13276 sql_identifier(&relation.schema_name),
13277 sql_identifier(&relation.table_name)
13278 )
13279}
13280
13281fn parse_key_role(value: &str) -> Result<ahri_tre_types::KeyRole, AppError> {
13282 match value.trim().to_ascii_lowercase().as_str() {
13283 "none" => Ok(ahri_tre_types::KeyRole::None),
13284 "record" => Ok(ahri_tre_types::KeyRole::Record),
13285 "external" => Ok(ahri_tre_types::KeyRole::External),
13286 _ => Err(AppError::Validation(format!(
13287 "variable keyrole is invalid: expected none, record, or external, got {value}"
13288 ))),
13289 }
13290}
13291
13292fn split_cli_list(value: &str) -> Vec<String> {
13293 value
13294 .split(',')
13295 .map(str::trim)
13296 .filter(|item| !item.is_empty())
13297 .map(str::to_string)
13298 .collect()
13299}
13300
13301fn parse_vocabulary_item_requests(
13302 items: &str,
13303) -> Result<Vec<RegisterVocabularyItemRequest>, AppError> {
13304 let mut parsed = Vec::new();
13305 for item in split_cli_list(items) {
13306 let (value, rest) = item.split_once([':', '=']).ok_or_else(|| {
13307 AppError::Validation(format!(
13308 "vocabulary item {item} is invalid: expected value:code[:description]"
13309 ))
13310 })?;
13311 let (code, description) = rest
13312 .split_once(':')
13313 .map(|(code, description)| (code, Some(description.to_string())))
13314 .unwrap_or((rest, None));
13315 let value = value.trim().parse::<i32>().map_err(|error| {
13316 AppError::Validation(format!(
13317 "vocabulary item {item} has invalid integer value: {error}"
13318 ))
13319 })?;
13320 let code = code.trim();
13321 if code.is_empty() {
13322 return Err(AppError::Validation(format!(
13323 "vocabulary item {item} has empty code"
13324 )));
13325 }
13326 parsed.push(RegisterVocabularyItemRequest {
13327 vocabulary_item_id: None,
13328 value,
13329 code: code.to_string(),
13330 description,
13331 });
13332 }
13333 if parsed.is_empty() {
13334 return Err(AppError::Validation(
13335 "vocabulary add requires at least one item".to_string(),
13336 ));
13337 }
13338 Ok(parsed)
13339}
13340
13341const STUDY_TYPE_ALIASES: &[(i64, &str)] = &[
13342 (1, "HDSS"),
13343 (2, "COHORT"),
13344 (3, "SURVEY"),
13345 (4, "PANEL"),
13346 (5, "CASE_CONTROL"),
13347 (6, "RCT"),
13348 (7, "QUASI_EXPERIMENTAL"),
13349 (8, "NATURAL_EXPERIMENT"),
13350 (10, "LAB_EXPERIMENT"),
13351 (11, "QUALITATIVE_INTERVIEW"),
13352 (12, "FOCUS_GROUP"),
13353 (13, "ETHNOGRAPHY"),
13354 (14, "PARTICIPATORY"),
13355 (15, "CASE_STUDY"),
13356 (16, "MIXED_METHODS"),
13357 (17, "SECONDARY_ANALYSIS"),
13358 (18, "DESK_REVIEW"),
13359 (19, "TIME_MOTION"),
13360 (20, "DIARY"),
13361 (21, "LONGITUDINAL_OBSERVATION"),
13362 (22, "SIMULATION"),
13363 (23, "AGENT_BASED_MODEL"),
13364 (24, "STATISTICAL_MODEL"),
13365 (25, "SYSTEM_DYNAMICS"),
13366 (26, "GENOMICS"),
13367 (27, "MULTIOMICS"),
13368 (28, "BIOBANK"),
13369 (29, "PHARMACOGENOMICS"),
13370];
13371
13372fn parse_study_type_id(value: &str) -> Result<ahri_tre_types::StudyTypeId, AppError> {
13373 let trimmed = value.trim();
13374 if trimmed.is_empty() {
13375 return Err(AppError::Validation(
13376 "study type is invalid: expected integer study type id or study type name".to_string(),
13377 ));
13378 }
13379 let id = match trimmed.parse::<i64>() {
13380 Ok(id) => id,
13381 Err(_) => {
13382 let normalized = trimmed
13383 .chars()
13384 .map(|character| match character {
13385 '-' | ' ' => '_',
13386 other => other.to_ascii_uppercase(),
13387 })
13388 .collect::<String>();
13389 STUDY_TYPE_ALIASES
13390 .iter()
13391 .find(|(_id, name)| *name == normalized.as_str())
13392 .map(|(id, _name)| *id)
13393 .ok_or_else(|| {
13394 let allowed = STUDY_TYPE_ALIASES
13395 .iter()
13396 .map(|(_id, name)| *name)
13397 .collect::<Vec<_>>()
13398 .join(", ");
13399 AppError::Validation(format!(
13400 "study type is invalid: expected integer study type id or one of: {allowed}"
13401 ))
13402 })?
13403 }
13404 };
13405 Ok(ahri_tre_types::StudyTypeId(id))
13406}
13407
13408fn noop_uuid(value: u128) -> uuid::Uuid {
13409 uuid::Uuid::from_u128(value)
13410}
13411
13412struct NoopDomainRepository;
13413struct NoopCatalogueRegistrationRepository;
13414struct NoopStudyRepository;
13415struct NoopStudyDomainRepository;
13416struct NoopStudyGovernanceRepository;
13417struct NoopAssetRepository;
13418struct NoopVariableRepository;
13419struct NoopVocabularyRepository;
13420struct NoopEntityRepository;
13421struct NoopTransformationRepository;
13422struct NoopTagRepository;
13423
13424#[async_trait::async_trait]
13425impl ahri_tre_core::CatalogueRegistrationRepository for NoopCatalogueRegistrationRepository {
13426 async fn create_domain_registration(
13427 &self,
13428 _domain: &ahri_tre_types::NewDomainRecord,
13429 _normalized_tags: &[String],
13430 ) -> Result<ahri_tre_types::DomainRecord, CoreError> {
13431 Err(CoreError::NotImplemented("create Domain registration"))
13432 }
13433
13434 async fn create_study_registration(
13435 &self,
13436 _study: &ahri_tre_types::NewStudyRecord,
13437 _domain_ids: &[ahri_tre_types::DomainId],
13438 _normalized_tags: &[String],
13439 ) -> Result<ahri_tre_types::StudyRecord, CoreError> {
13440 Err(CoreError::NotImplemented("create Study registration"))
13441 }
13442
13443 async fn save_domain_registration(
13444 &self,
13445 _domain: &ahri_tre_types::DomainRecord,
13446 _normalized_tags: &[String],
13447 ) -> Result<ahri_tre_types::DomainRecord, CoreError> {
13448 Err(CoreError::NotImplemented("save Domain registration"))
13449 }
13450
13451 async fn save_study_registration(
13452 &self,
13453 _study: &ahri_tre_types::StudyRecord,
13454 _domain_ids: &[ahri_tre_types::DomainId],
13455 _normalized_tags: &[String],
13456 ) -> Result<ahri_tre_types::StudyRecord, CoreError> {
13457 Err(CoreError::NotImplemented("save Study registration"))
13458 }
13459}
13460
13461#[async_trait::async_trait]
13462impl DomainRepository for NoopDomainRepository {
13463 async fn list_domains(&self) -> Result<Vec<ahri_tre_types::DomainRecord>, CoreError> {
13464 Ok(Vec::new())
13465 }
13466
13467 async fn get_domain_by_id(
13468 &self,
13469 _domain_id: ahri_tre_types::DomainId,
13470 ) -> Result<Option<ahri_tre_types::DomainRecord>, CoreError> {
13471 Ok(None)
13472 }
13473
13474 async fn get_domain_by_name(
13475 &self,
13476 _name: &str,
13477 ) -> Result<Option<ahri_tre_types::DomainRecord>, CoreError> {
13478 Ok(None)
13479 }
13480
13481 async fn create_domain(
13482 &self,
13483 domain: &ahri_tre_types::NewDomainRecord,
13484 ) -> Result<ahri_tre_types::DomainRecord, CoreError> {
13485 Ok(ahri_tre_types::DomainRecord {
13486 domain_id: ahri_tre_types::DomainId(1),
13487 name: domain.name.clone(),
13488 uri: domain.uri.clone(),
13489 description: domain.description.clone(),
13490 })
13491 }
13492
13493 async fn save_domain(
13494 &self,
13495 domain: &ahri_tre_types::DomainRecord,
13496 ) -> Result<ahri_tre_types::DomainRecord, CoreError> {
13497 Ok(domain.clone())
13498 }
13499}
13500
13501#[async_trait::async_trait]
13502impl StudyRepository for NoopStudyRepository {
13503 async fn list_studies(&self) -> Result<Vec<ahri_tre_types::StudyRecord>, CoreError> {
13504 Ok(Vec::new())
13505 }
13506
13507 async fn get_study_by_id(
13508 &self,
13509 _study_id: ahri_tre_types::StudyId,
13510 ) -> Result<Option<ahri_tre_types::StudyRecord>, CoreError> {
13511 Ok(None)
13512 }
13513
13514 async fn get_study_by_name(
13515 &self,
13516 _name: &str,
13517 ) -> Result<Option<ahri_tre_types::StudyRecord>, CoreError> {
13518 Ok(None)
13519 }
13520
13521 async fn create_study(
13522 &self,
13523 study: &ahri_tre_types::NewStudyRecord,
13524 ) -> Result<ahri_tre_types::StudyRecord, CoreError> {
13525 Ok(ahri_tre_types::StudyRecord {
13526 study_id: ahri_tre_types::StudyId(noop_uuid(1)),
13527 name: study.name.clone(),
13528 description: study.description.clone(),
13529 documentation: study.documentation.clone(),
13530 agent_instructions: study.agent_instructions.clone(),
13531 external_id: study.external_id.clone(),
13532 study_type_id: study.study_type_id,
13533 date_created: study.date_created,
13534 created_by: study.created_by.clone(),
13535 })
13536 }
13537
13538 async fn save_study(
13539 &self,
13540 study: &ahri_tre_types::StudyRecord,
13541 ) -> Result<ahri_tre_types::StudyRecord, CoreError> {
13542 Ok(study.clone())
13543 }
13544}
13545
13546#[async_trait::async_trait]
13547impl StudyDomainRepository for NoopStudyDomainRepository {
13548 async fn link_study_domain(
13549 &self,
13550 study_id: ahri_tre_types::StudyId,
13551 domain_id: ahri_tre_types::DomainId,
13552 ) -> Result<ahri_tre_types::StudyDomainLinkRecord, CoreError> {
13553 Ok(ahri_tre_types::StudyDomainLinkRecord {
13554 study_id,
13555 domain_id,
13556 })
13557 }
13558
13559 async fn list_domains_for_study(
13560 &self,
13561 _study_id: ahri_tre_types::StudyId,
13562 ) -> Result<Vec<ahri_tre_types::DomainRecord>, CoreError> {
13563 Ok(Vec::new())
13564 }
13565}
13566
13567#[async_trait::async_trait]
13568impl StudyGovernanceRepository for NoopStudyGovernanceRepository {
13569 async fn list_access_grants(
13570 &self,
13571 study_id: ahri_tre_types::StudyId,
13572 ) -> Result<Vec<ahri_tre_types::StudyAccessGrantRecord>, CoreError> {
13573 Ok(vec![
13574 ahri_tre_types::StudyAccessGrantRecord {
13575 study_id,
13576 user_id: "curator@example.org".to_string(),
13577 date_granted: None,
13578 granted_by: None,
13579 },
13580 ahri_tre_types::StudyAccessGrantRecord {
13581 study_id,
13582 user_id: "operator@example.org".to_string(),
13583 date_granted: None,
13584 granted_by: None,
13585 },
13586 ])
13587 }
13588
13589 async fn save_access_grant(
13590 &self,
13591 grant: &ahri_tre_types::StudyAccessGrantRecord,
13592 ) -> Result<ahri_tre_types::StudyAccessGrantRecord, CoreError> {
13593 Ok(grant.clone())
13594 }
13595
13596 async fn revoke_access_grant(
13597 &self,
13598 _study_id: ahri_tre_types::StudyId,
13599 _principal: &str,
13600 ) -> Result<(), CoreError> {
13601 Ok(())
13602 }
13603
13604 async fn revoke_study_access(
13605 &self,
13606 _study_id: ahri_tre_types::StudyId,
13607 _target_user_id: &str,
13608 ) -> Result<(), CoreError> {
13609 Ok(())
13610 }
13611
13612 async fn list_custodians(
13613 &self,
13614 study_id: ahri_tre_types::StudyId,
13615 ) -> Result<Vec<ahri_tre_types::StudyCustodianLinkRecord>, CoreError> {
13616 Ok(vec![
13617 ahri_tre_types::StudyCustodianLinkRecord {
13618 study_id,
13619 user_id: "curator@example.org".to_string(),
13620 is_primary: true,
13621 date_granted: None,
13622 granted_by: None,
13623 },
13624 ahri_tre_types::StudyCustodianLinkRecord {
13625 study_id,
13626 user_id: "operator@example.org".to_string(),
13627 is_primary: false,
13628 date_granted: None,
13629 granted_by: Some("curator@example.org".to_string()),
13630 },
13631 ])
13632 }
13633
13634 async fn transfer_primary_custodianship(
13635 &self,
13636 study_id: ahri_tre_types::StudyId,
13637 target_user_id: &str,
13638 ) -> Result<ahri_tre_types::StudyCustodianLinkRecord, CoreError> {
13639 Ok(ahri_tre_types::StudyCustodianLinkRecord {
13640 study_id,
13641 user_id: target_user_id.to_string(),
13642 is_primary: true,
13643 date_granted: None,
13644 granted_by: None,
13645 })
13646 }
13647
13648 async fn save_custodian(
13649 &self,
13650 custodian: &ahri_tre_types::StudyCustodianLinkRecord,
13651 ) -> Result<ahri_tre_types::StudyCustodianLinkRecord, CoreError> {
13652 Ok(custodian.clone())
13653 }
13654}
13655
13656#[async_trait::async_trait]
13657impl AssetRepository for NoopAssetRepository {
13658 async fn list_assets_for_study(
13659 &self,
13660 _study_id: ahri_tre_types::StudyId,
13661 ) -> Result<Vec<ahri_tre_types::AssetRecord>, CoreError> {
13662 Ok(Vec::new())
13663 }
13664
13665 async fn get_asset_by_id(
13666 &self,
13667 asset_id: ahri_tre_types::AssetId,
13668 ) -> Result<Option<ahri_tre_types::AssetRecord>, CoreError> {
13669 Ok(Some(ahri_tre_types::AssetRecord {
13670 asset_id,
13671 study_id: ahri_tre_types::StudyId(noop_uuid(1)),
13672 name: ahri_tre_types::NcName::parse("noop_asset")
13673 .expect("noop asset name should be valid"),
13674 description: None,
13675 agent_instructions: None,
13676 asset_type: ahri_tre_types::AssetType::Dataset,
13677 date_created: None,
13678 created_by: None,
13679 }))
13680 }
13681
13682 async fn get_asset_by_name(
13683 &self,
13684 _study_id: ahri_tre_types::StudyId,
13685 _name: &str,
13686 ) -> Result<Option<ahri_tre_types::AssetRecord>, CoreError> {
13687 Ok(None)
13688 }
13689
13690 async fn create_asset(
13691 &self,
13692 asset: &ahri_tre_types::NewAssetRecord,
13693 ) -> Result<ahri_tre_types::AssetRecord, CoreError> {
13694 Ok(ahri_tre_types::AssetRecord {
13695 asset_id: ahri_tre_types::AssetId(noop_uuid(2)),
13696 study_id: asset.study_id,
13697 name: asset.name.clone(),
13698 description: asset.description.clone(),
13699 agent_instructions: asset.agent_instructions.clone(),
13700 asset_type: asset.asset_type,
13701 date_created: asset.date_created,
13702 created_by: asset.created_by.clone(),
13703 })
13704 }
13705
13706 async fn save_asset(
13707 &self,
13708 asset: &ahri_tre_types::AssetRecord,
13709 ) -> Result<ahri_tre_types::AssetRecord, CoreError> {
13710 Ok(asset.clone())
13711 }
13712
13713 async fn list_asset_versions(
13714 &self,
13715 _asset_id: ahri_tre_types::AssetId,
13716 ) -> Result<Vec<ahri_tre_types::AssetVersionRecord>, CoreError> {
13717 Ok(Vec::new())
13718 }
13719
13720 async fn get_asset_version(
13721 &self,
13722 _version_id: ahri_tre_types::VersionId,
13723 ) -> Result<Option<ahri_tre_types::AssetVersionRecord>, CoreError> {
13724 Ok(None)
13725 }
13726
13727 async fn create_asset_version(
13728 &self,
13729 version: &ahri_tre_types::NewAssetVersionRecord,
13730 ) -> Result<ahri_tre_types::AssetVersionRecord, CoreError> {
13731 Ok(ahri_tre_types::AssetVersionRecord {
13732 version_id: ahri_tre_types::VersionId(noop_uuid(3)),
13733 asset_id: version.asset_id,
13734 major: version.major,
13735 minor: version.minor,
13736 patch: version.patch,
13737 version_label: version.version_label.clone(),
13738 version_note: version.version_note.clone(),
13739 is_latest: version.is_latest,
13740 doi: version.doi.clone(),
13741 created_at: version.created_at,
13742 created_by: version.created_by.clone(),
13743 })
13744 }
13745
13746 async fn save_asset_version(
13747 &self,
13748 version: &ahri_tre_types::AssetVersionRecord,
13749 ) -> Result<ahri_tre_types::AssetVersionRecord, CoreError> {
13750 Ok(version.clone())
13751 }
13752
13753 async fn list_datasets_for_asset(
13754 &self,
13755 _asset_id: ahri_tre_types::AssetId,
13756 ) -> Result<Vec<ahri_tre_types::DatasetRecord>, CoreError> {
13757 Ok(Vec::new())
13758 }
13759
13760 async fn get_dataset(
13761 &self,
13762 _dataset_id: ahri_tre_types::VersionId,
13763 ) -> Result<Option<ahri_tre_types::DatasetRecord>, CoreError> {
13764 Ok(None)
13765 }
13766
13767 async fn save_dataset(
13768 &self,
13769 dataset: &ahri_tre_types::DatasetRecord,
13770 ) -> Result<ahri_tre_types::DatasetRecord, CoreError> {
13771 Ok(dataset.clone())
13772 }
13773
13774 async fn list_datafiles_for_asset(
13775 &self,
13776 _asset_id: ahri_tre_types::AssetId,
13777 ) -> Result<Vec<ahri_tre_types::DataFileRecord>, CoreError> {
13778 Ok(Vec::new())
13779 }
13780
13781 async fn get_datafile(
13782 &self,
13783 _datafile_id: ahri_tre_types::VersionId,
13784 ) -> Result<Option<ahri_tre_types::DataFileRecord>, CoreError> {
13785 Ok(None)
13786 }
13787
13788 async fn save_datafile(
13789 &self,
13790 datafile: &ahri_tre_types::DataFileRecord,
13791 ) -> Result<ahri_tre_types::DataFileRecord, CoreError> {
13792 Ok(datafile.clone())
13793 }
13794}
13795
13796#[async_trait::async_trait]
13797impl VariableRepository for NoopVariableRepository {
13798 async fn list_value_types(&self) -> Result<Vec<ahri_tre_types::ValueTypeRecord>, CoreError> {
13799 Ok(default_value_type_records())
13800 }
13801
13802 async fn list_domain_variables(
13803 &self,
13804 _domain_id: ahri_tre_types::DomainId,
13805 ) -> Result<Vec<ahri_tre_types::VariableRecord>, CoreError> {
13806 Ok(Vec::new())
13807 }
13808
13809 async fn get_variable(
13810 &self,
13811 _variable_id: ahri_tre_types::VariableId,
13812 ) -> Result<Option<ahri_tre_types::VariableRecord>, CoreError> {
13813 Ok(None)
13814 }
13815
13816 async fn create_variable(
13817 &self,
13818 variable: &ahri_tre_types::NewVariableRecord,
13819 ) -> Result<ahri_tre_types::VariableRecord, CoreError> {
13820 Ok(ahri_tre_types::VariableRecord {
13821 variable_id: ahri_tre_types::VariableId(1),
13822 domain_id: variable.domain_id,
13823 name: variable.name.clone(),
13824 value_type_id: variable.value_type_id,
13825 value_format: variable.value_format.clone(),
13826 vocabulary_id: variable.vocabulary_id,
13827 key_role: variable.key_role,
13828 description: variable.description.clone(),
13829 note: variable.note.clone(),
13830 ontology_namespace: variable.ontology_namespace.clone(),
13831 ontology_class: variable.ontology_class.clone(),
13832 })
13833 }
13834
13835 async fn save_variable(
13836 &self,
13837 variable: &ahri_tre_types::VariableRecord,
13838 ) -> Result<ahri_tre_types::VariableRecord, CoreError> {
13839 Ok(variable.clone())
13840 }
13841
13842 async fn update_variable_metadata(
13843 &self,
13844 variable: &ahri_tre_types::VariableRecord,
13845 ) -> Result<ahri_tre_types::VariableRecord, CoreError> {
13846 Ok(variable.clone())
13847 }
13848
13849 async fn link_dataset_variable(
13850 &self,
13851 dataset_id: ahri_tre_types::VersionId,
13852 variable_id: ahri_tre_types::VariableId,
13853 row_role: ahri_tre_types::KeyRole,
13854 ) -> Result<ahri_tre_types::DatasetVariableLinkRecord, CoreError> {
13855 Ok(ahri_tre_types::DatasetVariableLinkRecord {
13856 dataset_id,
13857 variable_id,
13858 row_role,
13859 })
13860 }
13861
13862 async fn list_dataset_variables(
13863 &self,
13864 _dataset_id: ahri_tre_types::VersionId,
13865 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, CoreError> {
13866 Ok(Vec::new())
13867 }
13868
13869 async fn list_dataset_variables_for_variable(
13870 &self,
13871 _variable_id: ahri_tre_types::VariableId,
13872 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, CoreError> {
13873 Ok(Vec::new())
13874 }
13875}
13876
13877#[async_trait::async_trait]
13878impl VocabularyRepository for NoopVocabularyRepository {
13879 async fn list_vocabularies(
13880 &self,
13881 _domain_id: ahri_tre_types::DomainId,
13882 ) -> Result<Vec<ahri_tre_types::VocabularyRecord>, CoreError> {
13883 Ok(Vec::new())
13884 }
13885
13886 async fn get_vocabulary(
13887 &self,
13888 _vocabulary_id: ahri_tre_types::VocabularyId,
13889 ) -> Result<Option<ahri_tre_types::VocabularyRecord>, CoreError> {
13890 Ok(None)
13891 }
13892
13893 async fn create_vocabulary(
13894 &self,
13895 vocabulary: &ahri_tre_types::NewVocabularyRecord,
13896 ) -> Result<ahri_tre_types::VocabularyRecord, CoreError> {
13897 Ok(ahri_tre_types::VocabularyRecord {
13898 vocabulary_id: ahri_tre_types::VocabularyId(1),
13899 domain_id: vocabulary.domain_id,
13900 name: vocabulary.name.clone(),
13901 description: vocabulary.description.clone(),
13902 })
13903 }
13904
13905 async fn save_vocabulary(
13906 &self,
13907 vocabulary: &ahri_tre_types::VocabularyRecord,
13908 ) -> Result<ahri_tre_types::VocabularyRecord, CoreError> {
13909 Ok(vocabulary.clone())
13910 }
13911
13912 async fn update_vocabulary(
13913 &self,
13914 vocabulary: &ahri_tre_types::VocabularyRecord,
13915 ) -> Result<ahri_tre_types::VocabularyRecord, CoreError> {
13916 Ok(vocabulary.clone())
13917 }
13918
13919 async fn list_vocabulary_items(
13920 &self,
13921 _vocabulary_id: ahri_tre_types::VocabularyId,
13922 ) -> Result<Vec<ahri_tre_types::VocabularyItemRecord>, CoreError> {
13923 Ok(Vec::new())
13924 }
13925
13926 async fn get_vocabulary_item(
13927 &self,
13928 _vocabulary_item_id: ahri_tre_types::VocabularyItemId,
13929 ) -> Result<Option<ahri_tre_types::VocabularyItemRecord>, CoreError> {
13930 Ok(None)
13931 }
13932
13933 async fn create_vocabulary_items(
13934 &self,
13935 vocabulary_id: ahri_tre_types::VocabularyId,
13936 items: &[ahri_tre_types::NewVocabularyItemRecord],
13937 ) -> Result<Vec<ahri_tre_types::VocabularyItemRecord>, CoreError> {
13938 Ok(items
13939 .iter()
13940 .enumerate()
13941 .map(|(index, item)| ahri_tre_types::VocabularyItemRecord {
13942 vocabulary_item_id: ahri_tre_types::VocabularyItemId(index as i64 + 1),
13943 vocabulary_id,
13944 value: item.value,
13945 code: item.code.clone(),
13946 description: item.description.clone(),
13947 })
13948 .collect())
13949 }
13950
13951 async fn save_vocabulary_items(
13952 &self,
13953 _items: &[ahri_tre_types::VocabularyItemRecord],
13954 ) -> Result<(), CoreError> {
13955 Ok(())
13956 }
13957
13958 async fn update_vocabulary_items(
13959 &self,
13960 _items: &[ahri_tre_types::VocabularyItemRecord],
13961 ) -> Result<(), CoreError> {
13962 Ok(())
13963 }
13964
13965 async fn list_vocabulary_mappings(
13966 &self,
13967 ) -> Result<Vec<ahri_tre_types::VocabularyMappingRecord>, CoreError> {
13968 Ok(Vec::new())
13969 }
13970
13971 async fn create_vocabulary_mapping(
13972 &self,
13973 mapping: &ahri_tre_types::NewVocabularyMappingRecord,
13974 ) -> Result<ahri_tre_types::VocabularyMappingRecord, CoreError> {
13975 Ok(ahri_tre_types::VocabularyMappingRecord {
13976 vocabulary_mapping_id: 1,
13977 from_vocabulary_item: mapping.from_vocabulary_item,
13978 to_vocabulary_item: mapping.to_vocabulary_item,
13979 })
13980 }
13981
13982 async fn save_vocabulary_mapping(
13983 &self,
13984 mapping: &ahri_tre_types::VocabularyMappingRecord,
13985 ) -> Result<ahri_tre_types::VocabularyMappingRecord, CoreError> {
13986 Ok(mapping.clone())
13987 }
13988}
13989
13990#[async_trait::async_trait]
13991impl EntityRepository for NoopEntityRepository {
13992 async fn create_entity(
13993 &self,
13994 entity: &ahri_tre_types::NewEntityRecord,
13995 ) -> Result<ahri_tre_types::EntityRecord, CoreError> {
13996 Ok(ahri_tre_types::EntityRecord {
13997 entity_id: ahri_tre_types::EntityId(1),
13998 domain_id: entity.domain_id,
13999 uuid: Some(noop_uuid(4)),
14000 name: entity.name.clone(),
14001 description: entity.description.clone(),
14002 agent_instructions: entity.agent_instructions.clone(),
14003 ontology_namespace: entity.ontology_namespace.clone(),
14004 ontology_class: entity.ontology_class.clone(),
14005 })
14006 }
14007
14008 async fn save_entity(
14009 &self,
14010 entity: &ahri_tre_types::EntityRecord,
14011 ) -> Result<ahri_tre_types::EntityRecord, CoreError> {
14012 Ok(entity.clone())
14013 }
14014
14015 async fn create_entity_relation(
14016 &self,
14017 relation: &ahri_tre_types::NewEntityRelationRecord,
14018 ) -> Result<ahri_tre_types::EntityRelationRecord, CoreError> {
14019 Ok(ahri_tre_types::EntityRelationRecord {
14020 entity_relation_id: ahri_tre_types::EntityRelationId(1),
14021 subject_entity_id: relation.subject_entity_id,
14022 object_entity_id: relation.object_entity_id,
14023 domain_id: relation.domain_id,
14024 uuid: Some(noop_uuid(5)),
14025 name: relation.name.clone(),
14026 description: relation.description.clone(),
14027 agent_instructions: relation.agent_instructions.clone(),
14028 ontology_namespace: relation.ontology_namespace.clone(),
14029 ontology_class: relation.ontology_class.clone(),
14030 })
14031 }
14032
14033 async fn save_entity_relation(
14034 &self,
14035 relation: &ahri_tre_types::EntityRelationRecord,
14036 ) -> Result<ahri_tre_types::EntityRelationRecord, CoreError> {
14037 Ok(relation.clone())
14038 }
14039}
14040
14041#[async_trait::async_trait]
14042impl TransformationRepository for NoopTransformationRepository {
14043 async fn create_transformation(
14044 &self,
14045 transformation: &ahri_tre_types::NewTransformationRecord,
14046 ) -> Result<ahri_tre_types::TransformationRecord, CoreError> {
14047 Ok(ahri_tre_types::TransformationRecord {
14048 transformation_id: ahri_tre_types::TransformationId(1),
14049 transformation_type: transformation.transformation_type,
14050 description: transformation.description.clone(),
14051 repository_url: transformation.repository_url.clone(),
14052 commit_hash: transformation.commit_hash.clone(),
14053 file_path: transformation.file_path.clone(),
14054 date_created: transformation.date_created,
14055 created_by: transformation.created_by.clone(),
14056 })
14057 }
14058
14059 async fn save_transformation(
14060 &self,
14061 transformation: &ahri_tre_types::TransformationRecord,
14062 ) -> Result<ahri_tre_types::TransformationRecord, CoreError> {
14063 Ok(transformation.clone())
14064 }
14065
14066 async fn add_inputs(
14067 &self,
14068 _transformation_id: ahri_tre_types::TransformationId,
14069 _input_ids: &[ahri_tre_types::VersionId],
14070 ) -> Result<(), CoreError> {
14071 Ok(())
14072 }
14073
14074 async fn add_outputs(
14075 &self,
14076 _transformation_id: ahri_tre_types::TransformationId,
14077 _output_ids: &[ahri_tre_types::VersionId],
14078 ) -> Result<(), CoreError> {
14079 Ok(())
14080 }
14081
14082 async fn list_inputs(
14083 &self,
14084 _transformation_id: ahri_tre_types::TransformationId,
14085 ) -> Result<Vec<ahri_tre_types::TransformationInputLinkRecord>, CoreError> {
14086 Ok(Vec::new())
14087 }
14088
14089 async fn list_outputs(
14090 &self,
14091 _transformation_id: ahri_tre_types::TransformationId,
14092 ) -> Result<Vec<ahri_tre_types::TransformationOutputLinkRecord>, CoreError> {
14093 Ok(Vec::new())
14094 }
14095
14096 async fn list_transformations_producing(
14097 &self,
14098 _version_id: ahri_tre_types::VersionId,
14099 ) -> Result<Vec<ahri_tre_types::TransformationRecord>, CoreError> {
14100 Ok(Vec::new())
14101 }
14102
14103 async fn list_transformations(
14104 &self,
14105 ) -> Result<Vec<ahri_tre_types::TransformationRecord>, CoreError> {
14106 Ok(Vec::new())
14107 }
14108}
14109
14110#[async_trait::async_trait]
14111impl TagRepository for NoopTagRepository {
14112 async fn list_tags(&self) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
14113 Ok(Vec::new())
14114 }
14115
14116 async fn upsert_tag(
14117 &self,
14118 normalized_name: &str,
14119 ) -> Result<ahri_tre_types::TagRecord, CoreError> {
14120 Ok(ahri_tre_types::TagRecord {
14121 tag_id: ahri_tre_types::TagId(0),
14122 name: normalized_name.to_string(),
14123 })
14124 }
14125
14126 async fn attach_tag(
14127 &self,
14128 _tag_id: ahri_tre_types::TagId,
14129 _target: TagAttachmentTarget,
14130 ) -> Result<(), CoreError> {
14131 Ok(())
14132 }
14133
14134 async fn detach_tag(
14135 &self,
14136 _tag_id: ahri_tre_types::TagId,
14137 _target: TagAttachmentTarget,
14138 ) -> Result<(), CoreError> {
14139 Ok(())
14140 }
14141
14142 async fn list_tags_for_target(
14143 &self,
14144 _target: TagAttachmentTarget,
14145 ) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
14146 Ok(Vec::new())
14147 }
14148}
14149
14150#[cfg(test)]
14151mod tests {
14152 use super::*;
14153 use ahri_tre_core::{
14154 ArchiveMode, AssetRepository, CatalogueRegistrationRepository, DomainRepository,
14155 EntityRepository, StudyDomainRepository, StudyGovernanceRepository, StudyRepository,
14156 TagRepository, TransformationRepository, VariableRepository, VocabularyRepository,
14157 };
14158 use ahri_tre_runtime::{DataStoreRuntime, DuckLakeConnection, PgStoreConnection};
14159 use ahri_tre_types::{
14160 AssetId, AssetRecord, AssetType, AssetVersionDuoRestrictionId,
14161 AssetVersionDuoRestrictionRecord, AssetVersionEntityRecord, AssetVersionRecord,
14162 AssetVersionRelationInstanceRecord, DataFileRecord, DataStoreProfile, DatasetRecord,
14163 DatasetVersionEntityRecord, DatasetVersionRelationInstanceRecord, DomainId, DomainRecord,
14164 EncryptionMode, EntityId, EntityInstanceRecord, EntityInstanceWriteRecord, EntityRecord,
14165 EntityRelationId, EntityRelationRecord, KeyRole, LakeAttachProfile, LakeAuthProfile,
14166 NcName, NewAssetRecord, NewAssetVersionDuoRestrictionRecord, NewAssetVersionRecord,
14167 NewDomainRecord, NewEntityInstanceRecord, NewEntityRecord, NewEntityRelationRecord,
14168 NewRelationInstanceRecord, NewStudyDuoRestrictionRecord, NewStudyRecord, NewTagRecord,
14169 NewTransformationRecord, RelationInstanceRecord, RelationInstanceWriteRecord,
14170 StudyAccessGrantRecord, StudyCustodianLinkRecord, StudyDomainLinkRecord,
14171 StudyDuoRestrictionId, StudyDuoRestrictionRecord, StudyEntityInstanceRecord, StudyId,
14172 StudyRecord, StudyRelationInstanceRecord, TagId, TransformationId,
14173 TransformationInputLinkRecord, TransformationOutputLinkRecord, TransformationRecord,
14174 TransformationType, ValueTypeId, VariableId, VariableRecord, VersionId, VocabularyId,
14175 VocabularyItemId, VocabularyItemRecord, VocabularyRecord,
14176 };
14177 use async_trait::async_trait;
14178 use openssl::asn1::Asn1Time;
14179 use openssl::bn::{BigNum, MsbOption};
14180 use openssl::hash::MessageDigest;
14181 use openssl::pkey::{PKey, Private};
14182 use openssl::rsa::Rsa;
14183 use openssl::ssl::{SslAcceptor, SslMethod};
14184 use openssl::x509::extension::{BasicConstraints, KeyUsage, SubjectAlternativeName};
14185 use openssl::x509::{X509, X509NameBuilder};
14186 use std::collections::BTreeMap;
14187 use std::fs;
14188 use std::io::{Read, Write};
14189 use std::net::TcpListener;
14190 use std::ops::Deref;
14191 use std::os::unix::fs::PermissionsExt;
14192 use std::path::{Path, PathBuf};
14193
14194 use std::process::Command;
14195 use std::sync::{Arc, Mutex, MutexGuard};
14196 use std::thread;
14197 use std::time::Duration as StdDuration;
14198
14199 fn uuid(value: u128) -> uuid::Uuid {
14200 uuid::Uuid::from_u128(value)
14201 }
14202
14203 #[derive(Clone)]
14204 struct ArchiveDeleteGovernanceFixture {
14205 custodians: Vec<StudyCustodianLinkRecord>,
14206 list_error: Option<String>,
14207 }
14208
14209 #[async_trait]
14210 impl StudyGovernanceRepository for ArchiveDeleteGovernanceFixture {
14211 async fn list_access_grants(
14212 &self,
14213 _study_id: StudyId,
14214 ) -> Result<Vec<StudyAccessGrantRecord>, CoreError> {
14215 Ok(Vec::new())
14216 }
14217
14218 async fn save_access_grant(
14219 &self,
14220 grant: &StudyAccessGrantRecord,
14221 ) -> Result<StudyAccessGrantRecord, CoreError> {
14222 Ok(grant.clone())
14223 }
14224
14225 async fn revoke_study_access(
14226 &self,
14227 _study_id: StudyId,
14228 _target_user_id: &str,
14229 ) -> Result<(), CoreError> {
14230 Ok(())
14231 }
14232
14233 async fn list_custodians(
14234 &self,
14235 _study_id: StudyId,
14236 ) -> Result<Vec<StudyCustodianLinkRecord>, CoreError> {
14237 if let Some(error) = &self.list_error {
14238 return Err(CoreError::Infrastructure(error.clone()));
14239 }
14240 Ok(self.custodians.clone())
14241 }
14242
14243 async fn transfer_primary_custodianship(
14244 &self,
14245 study_id: StudyId,
14246 target_user_id: &str,
14247 ) -> Result<StudyCustodianLinkRecord, CoreError> {
14248 Ok(StudyCustodianLinkRecord {
14249 study_id,
14250 user_id: target_user_id.to_string(),
14251 is_primary: true,
14252 date_granted: None,
14253 granted_by: None,
14254 })
14255 }
14256
14257 async fn save_custodian(
14258 &self,
14259 custodian: &StudyCustodianLinkRecord,
14260 ) -> Result<StudyCustodianLinkRecord, CoreError> {
14261 Ok(custodian.clone())
14262 }
14263 }
14264
14265 fn archive_delete_study_request(actor: Option<&str>) -> PlanArchiveDeleteRequest {
14266 PlanArchiveDeleteRequest {
14267 target: DeleteTarget::Study(StudyId(uuid(8))),
14268 archive_mode: ArchiveMode::StudyArchive,
14269 cascade: false,
14270 force: false,
14271 reason: "obsolete governed study".to_string(),
14272 actor: actor.map(str::to_string),
14273 version_target: None,
14274 delete_all_versions: false,
14275 dry_run: true,
14276 }
14277 }
14278
14279 fn archive_delete_service_with_governance(
14280 governance: ArchiveDeleteGovernanceFixture,
14281 ) -> AppService {
14282 AppService::new(AppRepositories {
14283 registrations: Arc::new(NoopCatalogueRegistrationRepository),
14284 domains: Arc::new(NoopDomainRepository),
14285 studies: Arc::new(NoopStudyRepository),
14286 study_domains: Arc::new(NoopStudyDomainRepository),
14287 study_governance: Arc::new(governance),
14288 assets: Arc::new(NoopAssetRepository),
14289 variables: Arc::new(NoopVariableRepository),
14290 vocabularies: Arc::new(NoopVocabularyRepository),
14291 entities: Arc::new(NoopEntityRepository),
14292 transformations: Arc::new(NoopTransformationRepository),
14293 tags: Arc::new(NoopTagRepository),
14294 })
14295 }
14296
14297 #[test]
14298 fn archive_delete_planning_accepts_study_custodian_actor() {
14299 let study_id = StudyId(uuid(8));
14300 let service = archive_delete_service_with_governance(ArchiveDeleteGovernanceFixture {
14301 custodians: vec![StudyCustodianLinkRecord {
14302 study_id,
14303 user_id: "curator@example.org".to_string(),
14304 is_primary: false,
14305 date_granted: None,
14306 granted_by: Some("primary@example.org".to_string()),
14307 }],
14308 list_error: None,
14309 });
14310
14311 let result = block_on(
14312 service.plan_archive_delete(archive_delete_study_request(Some("curator@example.org"))),
14313 )
14314 .unwrap();
14315
14316 assert!(result.accepted);
14317 let plan = result.plan.unwrap();
14318 assert_eq!(
14319 plan.authorization.status,
14320 ArchiveDeleteAuthorizationStatus::Authorized
14321 );
14322 assert_eq!(plan.authorization.principal, "curator@example.org");
14323 assert_eq!(plan.actor.as_deref(), Some("curator@example.org"));
14324 }
14325
14326 #[test]
14327 fn archive_delete_planning_rejects_missing_and_unauthorized_study_actor() {
14328 let study_id = StudyId(uuid(8));
14329 let service = archive_delete_service_with_governance(ArchiveDeleteGovernanceFixture {
14330 custodians: vec![StudyCustodianLinkRecord {
14331 study_id,
14332 user_id: "curator@example.org".to_string(),
14333 is_primary: true,
14334 date_granted: None,
14335 granted_by: None,
14336 }],
14337 list_error: None,
14338 });
14339
14340 let missing_actor =
14341 block_on(service.plan_archive_delete(archive_delete_study_request(None))).unwrap();
14342 assert!(
14343 missing_actor
14344 .validation_errors
14345 .iter()
14346 .any(|error| error.code == "missing_actor")
14347 );
14348
14349 let unauthorized = block_on(
14350 service.plan_archive_delete(archive_delete_study_request(Some("reader@example.org"))),
14351 )
14352 .unwrap();
14353 assert!(!unauthorized.accepted);
14354 assert!(
14355 unauthorized
14356 .validation_errors
14357 .iter()
14358 .any(|error| error.code == "unauthorized_actor")
14359 );
14360 }
14361
14362 #[test]
14363 fn archive_delete_planning_redacts_authorization_repository_errors() {
14364 let service = archive_delete_service_with_governance(ArchiveDeleteGovernanceFixture {
14365 custodians: Vec::new(),
14366 list_error: Some(
14367 "permission denied reading /home/operator/.pgpass password=hunter2 token=abc https://example.test/archive?X-Amz-Signature=secret"
14368 .to_string(),
14369 ),
14370 });
14371
14372 let result = block_on(
14373 service.plan_archive_delete(archive_delete_study_request(Some("curator@example.org"))),
14374 )
14375 .unwrap();
14376
14377 assert!(!result.accepted);
14378 let message = &result.validation_errors[0].message;
14379 assert!(message.contains("authorization could not be evaluated"));
14380 assert!(!message.contains("/home/operator"));
14381 assert!(!message.contains("hunter2"));
14382 assert!(!message.contains("token=abc"));
14383 assert!(!message.contains("X-Amz-Signature"));
14384 }
14385
14386 #[test]
14387 fn study_archive_preparation_requires_reason_and_actor() {
14388 let repositories = AssetWorkflowRepositories::new();
14389 let service = repositories.service();
14390 let study = test_study("archive_prepare_validation_study");
14391 repositories.seed_study(study.clone());
14392
14393 let missing_reason = block_on(service.plan_or_prepare_study_archive_for_deletion(
14394 PrepareStudyArchiveForDeletionRequest {
14395 study_id: study.study_id,
14396 reason: " ".to_string(),
14397 actor: Some("curator@example.org".to_string()),
14398 },
14399 ))
14400 .expect_err("blank reason should fail");
14401 assert!(
14402 matches!(missing_reason, AppError::Validation(message) if message.contains("reason"))
14403 );
14404
14405 let missing_actor = block_on(service.plan_or_prepare_study_archive_for_deletion(
14406 PrepareStudyArchiveForDeletionRequest {
14407 study_id: study.study_id,
14408 reason: "retire duplicate study".to_string(),
14409 actor: Some(" ".to_string()),
14410 },
14411 ))
14412 .expect_err("blank actor should fail");
14413 assert!(
14414 matches!(missing_actor, AppError::Validation(message) if message.contains("actor"))
14415 );
14416 }
14417
14418 #[test]
14419 fn study_archive_preparation_requires_existing_source_study() {
14420 let service = AssetWorkflowRepositories::new().service();
14421
14422 let error = block_on(service.plan_or_prepare_study_archive_for_deletion(
14423 PrepareStudyArchiveForDeletionRequest {
14424 study_id: StudyId(uuid(404)),
14425 reason: "retire duplicate study".to_string(),
14426 actor: Some("curator@example.org".to_string()),
14427 },
14428 ))
14429 .expect_err("missing study should fail");
14430
14431 assert!(
14432 matches!(error, AppError::Validation(message) if message.contains("study not found"))
14433 );
14434 }
14435
14436 #[test]
14437 fn study_archive_preparation_ensures_archive_study_without_deleting_or_creating_payloads() {
14438 let repositories = AssetWorkflowRepositories::new();
14439 let service = repositories.service();
14440 let source_study = test_study("archive_prepare_source_study");
14441 repositories.seed_study(source_study.clone());
14442
14443 let prepared = block_on(service.plan_or_prepare_study_archive_for_deletion(
14444 PrepareStudyArchiveForDeletionRequest {
14445 study_id: source_study.study_id,
14446 reason: "retire duplicate study".to_string(),
14447 actor: Some("curator@example.org".to_string()),
14448 },
14449 ))
14450 .expect("archive preparation should succeed");
14451
14452 assert_eq!(prepared.source_study, source_study);
14453 assert_eq!(prepared.archive_domain.name.as_str(), "Archive");
14454 assert_eq!(prepared.archive_study.name.as_str(), "Archive");
14455 assert_eq!(prepared.archive_datafile.archive_asset_id, None);
14456 assert_eq!(prepared.archive_datafile.archive_version_id, None);
14457 assert_eq!(prepared.archive_datafile.archive_datafile_id, None);
14458 assert_eq!(
14459 prepared.archive_datafile.mime_type,
14460 "application/x-tar+zstd"
14461 );
14462 assert_eq!(
14463 prepared.archive_datafile.dataset_payload_format,
14464 DatasetExportFormat::Parquet
14465 );
14466 assert!(!prepared.source_study_delete_started);
14467 assert!(!prepared.lake_payload_created);
14468 assert_eq!(
14469 block_on(repositories.studies.get_study_by_id(source_study.study_id))
14470 .expect("source study lookup should succeed"),
14471 Some(source_study)
14472 );
14473 assert!(
14474 repositories
14475 .state
14476 .assets_by_id
14477 .lock()
14478 .expect("asset lock should not be poisoned")
14479 .is_empty()
14480 );
14481 assert!(
14482 repositories
14483 .state
14484 .datafiles_by_version
14485 .lock()
14486 .expect("datafile lock should not be poisoned")
14487 .is_empty()
14488 );
14489 }
14490
14491 #[test]
14492 fn study_archive_preparation_reuses_existing_archive_domain_and_study() {
14493 let repositories = AssetWorkflowRepositories::new();
14494 let service = repositories.service();
14495 let source_study = test_study("archive_prepare_reuse_source");
14496 let archive_domain = test_domain(702, "Archive");
14497 let archive_study = StudyRecord {
14498 study_id: StudyId(uuid(703)),
14499 name: nc_name("Archive"),
14500 description: Some("Existing archive study".to_string()),
14501 documentation: None,
14502 agent_instructions: None,
14503 external_id: Some("Archive".to_string()),
14504 study_type_id: None,
14505 date_created: None,
14506 created_by: Some("bootstrap@example.org".to_string()),
14507 };
14508 repositories.seed_study(source_study.clone());
14509 repositories.seed_domain(archive_domain.clone());
14510 repositories.seed_study(archive_study.clone());
14511
14512 let prepared = block_on(service.plan_or_prepare_study_archive_for_deletion(
14513 PrepareStudyArchiveForDeletionRequest {
14514 study_id: source_study.study_id,
14515 reason: "retire duplicate study".to_string(),
14516 actor: Some("curator@example.org".to_string()),
14517 },
14518 ))
14519 .expect("archive preparation should reuse existing Archive rows");
14520
14521 assert_eq!(prepared.archive_domain.domain_id, archive_domain.domain_id);
14522 assert_eq!(prepared.archive_study.study_id, archive_study.study_id);
14523 let study_count = block_on(repositories.studies.list_studies())
14524 .expect("study list should read")
14525 .len();
14526 assert_eq!(study_count, 2);
14527 }
14528
14529 #[test]
14530 fn archive_study_for_deletion_ingests_bundle_into_archive_study_without_source_delete() {
14531 let repositories = AssetWorkflowRepositories::new();
14532 let service = repositories.service();
14533 let source_study = test_study("archive_bundle_source");
14534 repositories.seed_study(source_study.clone());
14535 let lake_root = temp_lake_root_path();
14536 let mut session = test_lake_session(&lake_root);
14537
14538 let archived = block_on(service.archive_study_for_deletion(
14539 &mut session,
14540 ArchiveStudyForDeletionRequest {
14541 study_id: source_study.study_id,
14542 reason: "retire duplicate empty study".to_string(),
14543 actor: Some("curator@example.org".to_string()),
14544 overwrite_existing_exports: true,
14545 },
14546 ))
14547 .expect("empty study archive should be ingested");
14548
14549 assert_eq!(archived.manifest.target.study_id, source_study.study_id);
14550 assert_eq!(
14551 archived.manifest.dataset_payload_format,
14552 DatasetExportFormat::Parquet
14553 );
14554 assert_eq!(
14555 archived.archive_asset.study_id,
14556 archived.prepared.archive_study.study_id
14557 );
14558 assert_eq!(
14559 archived.archive_version.asset_id,
14560 archived.archive_asset.asset_id
14561 );
14562 assert_eq!(
14563 archived.archive_datafile.datafile_id,
14564 archived.archive_version.version_id
14565 );
14566 assert_eq!(
14567 archived.archive_datafile.edam_format,
14568 "application/x-tar+zstd"
14569 );
14570 assert_eq!(archived.archive_datafile.compressed, Some(false));
14571 assert!(!archived.source_study_delete_started);
14572 assert!(archived.source_size_bytes > 0);
14573 assert!(archived.stored_size_bytes > 0);
14574 assert_eq!(
14575 block_on(repositories.studies.get_study_by_id(source_study.study_id))
14576 .expect("source study lookup should succeed"),
14577 Some(source_study)
14578 );
14579 assert!(
14580 datafile_storage_uri_path(
14581 &lake_root.display().to_string(),
14582 &archived.archive_datafile.storage_uri
14583 )
14584 .expect("archive datafile URI should resolve")
14585 .exists()
14586 );
14587 }
14588
14589 #[test]
14590 fn archive_study_for_deletion_rejects_before_bundle_when_source_missing() {
14591 let repositories = AssetWorkflowRepositories::new();
14592 let service = repositories.service();
14593 let lake_root = temp_lake_root_path();
14594 let mut session = test_lake_session(&lake_root);
14595
14596 let error = block_on(service.archive_study_for_deletion(
14597 &mut session,
14598 ArchiveStudyForDeletionRequest {
14599 study_id: StudyId(uuid(909_001)),
14600 reason: "retire missing study".to_string(),
14601 actor: Some("curator@example.org".to_string()),
14602 overwrite_existing_exports: true,
14603 },
14604 ))
14605 .expect_err("missing source study should fail before archive ingest");
14606
14607 assert!(
14608 matches!(error, AppError::Validation(message) if message.contains("study not found"))
14609 );
14610 assert!(
14611 repositories
14612 .state
14613 .assets_by_id
14614 .lock()
14615 .expect("asset lock should not be poisoned")
14616 .is_empty()
14617 );
14618 assert!(
14619 repositories
14620 .state
14621 .datafiles_by_version
14622 .lock()
14623 .expect("datafile lock should not be poisoned")
14624 .is_empty()
14625 );
14626 }
14627
14628 #[test]
14629 fn archive_study_for_deletion_preserves_source_when_archive_study_creation_fails() {
14630 let repositories = AssetWorkflowRepositories::new();
14631 let service = repositories.service();
14632 let source_study = test_study("archive_bundle_archive_study_failure_source");
14633 repositories.seed_study(source_study.clone());
14634 repositories.fail_next_create_study("issue02 injected Archive study creation failure");
14635 let lake_root = temp_lake_root_path();
14636 let mut session = test_lake_session(&lake_root);
14637
14638 let error = block_on(service.archive_study_for_deletion(
14639 &mut session,
14640 ArchiveStudyForDeletionRequest {
14641 study_id: source_study.study_id,
14642 reason: "retire study".to_string(),
14643 actor: Some("curator@example.org".to_string()),
14644 overwrite_existing_exports: true,
14645 },
14646 ))
14647 .expect_err("Archive study creation failure should abort archive workflow");
14648
14649 assert!(
14650 matches!(error, AppError::Infrastructure(message) if message.contains("issue02 injected Archive study creation failure"))
14651 );
14652 assert_eq!(
14653 block_on(repositories.studies.get_study_by_id(source_study.study_id))
14654 .expect("source study lookup should succeed"),
14655 Some(source_study)
14656 );
14657 assert!(
14658 repositories
14659 .state
14660 .datafiles_by_version
14661 .lock()
14662 .expect("datafile lock should not be poisoned")
14663 .is_empty()
14664 );
14665 }
14666
14667 #[test]
14668 fn archive_study_for_deletion_preserves_source_when_bundle_metadata_collection_fails() {
14669 let repositories = AssetWorkflowRepositories::new();
14670 let service = repositories.service();
14671 let source_study = test_study("archive_bundle_metadata_failure_source");
14672 let source_domain = test_domain(810_001, "archive_bundle_metadata_failure_domain");
14673 repositories.seed_study(source_study.clone());
14674 repositories.seed_domain(source_domain.clone());
14675 repositories.seed_study_domain(source_study.study_id, source_domain.clone());
14676 repositories.fail_next_list_domain_variables("issue02 injected metadata listing failure");
14677 let lake_root = temp_lake_root_path();
14678 let mut session = test_lake_session(&lake_root);
14679
14680 let error = block_on(service.archive_study_for_deletion(
14681 &mut session,
14682 ArchiveStudyForDeletionRequest {
14683 study_id: source_study.study_id,
14684 reason: "retire study".to_string(),
14685 actor: Some("curator@example.org".to_string()),
14686 overwrite_existing_exports: true,
14687 },
14688 ))
14689 .expect_err("metadata collection failure should abort before bundle ingest");
14690
14691 assert!(
14692 matches!(error, AppError::Infrastructure(message) if message.contains("list study archive variables"))
14693 );
14694 assert_eq!(
14695 block_on(repositories.studies.get_study_by_id(source_study.study_id))
14696 .expect("source study lookup should succeed"),
14697 Some(source_study)
14698 );
14699 assert!(
14700 repositories
14701 .state
14702 .datafiles_by_version
14703 .lock()
14704 .expect("datafile lock should not be poisoned")
14705 .is_empty()
14706 );
14707 }
14708
14709 #[test]
14710 fn archive_study_for_deletion_preserves_source_when_archive_datafile_ingest_fails() {
14711 let repositories = AssetWorkflowRepositories::new();
14712 let service = repositories.service();
14713 let source_study = test_study("archive_bundle_ingest_failure_source");
14714 repositories.seed_study(source_study.clone());
14715 repositories.fail_next_save_datafile("issue02 injected Archive datafile save failure");
14716 let lake_root = temp_lake_root_path();
14717 let mut session = test_lake_session(&lake_root);
14718
14719 let error = block_on(service.archive_study_for_deletion(
14720 &mut session,
14721 ArchiveStudyForDeletionRequest {
14722 study_id: source_study.study_id,
14723 reason: "retire study".to_string(),
14724 actor: Some("curator@example.org".to_string()),
14725 overwrite_existing_exports: true,
14726 },
14727 ))
14728 .expect_err("Archive datafile ingest failure should abort archive workflow");
14729
14730 assert!(
14731 matches!(error, AppError::Infrastructure(message) if message.contains("save Archive-study archive datafile"))
14732 );
14733 assert_eq!(
14734 block_on(repositories.studies.get_study_by_id(source_study.study_id))
14735 .expect("source study lookup should succeed"),
14736 Some(source_study.clone())
14737 );
14738 assert!(
14739 repositories
14740 .state
14741 .datafiles_by_version
14742 .lock()
14743 .expect("datafile lock should not be poisoned")
14744 .is_empty()
14745 );
14746 let retry = block_on(service.archive_study_for_deletion(
14747 &mut session,
14748 ArchiveStudyForDeletionRequest {
14749 study_id: source_study.study_id,
14750 reason: "retry archive after storage recovery".into(),
14751 actor: Some("curator@example.org".into()),
14752 overwrite_existing_exports: true,
14753 },
14754 ))
14755 .expect("a failed archive registration must not block retry");
14756 assert!(!retry.source_study_delete_started);
14757 assert_eq!(retry.prepared.source_study.study_id, source_study.study_id);
14758 }
14759
14760 struct TestHttpsRootGuard {
14761 _lock: MutexGuard<'static, ()>,
14762 }
14763
14764 impl Drop for TestHttpsRootGuard {
14765 fn drop(&mut self) {
14766 *TEST_HTTPS_ROOT_CERTIFICATE_DER
14767 .lock()
14768 .expect("test HTTPS certificate lock should not be poisoned") = None;
14769 *TEST_HTTPS_MAX_BYTES
14770 .lock()
14771 .expect("test HTTPS max bytes lock should not be poisoned") = None;
14772 *TEST_HTTPS_TIMEOUT
14773 .lock()
14774 .expect("test HTTPS timeout lock should not be poisoned") = None;
14775 }
14776 }
14777
14778 #[test]
14779 fn datastore_content_counts_reports_empty_supported_categories() {
14780 let service = AppService::new(AppRepositories {
14781 registrations: Arc::new(NoopCatalogueRegistrationRepository),
14782 domains: Arc::new(NoopDomainRepository),
14783 studies: Arc::new(NoopStudyRepository),
14784 study_domains: Arc::new(NoopStudyDomainRepository),
14785 study_governance: Arc::new(NoopStudyGovernanceRepository),
14786 assets: Arc::new(NoopAssetRepository),
14787 variables: Arc::new(NoopVariableRepository),
14788 vocabularies: Arc::new(NoopVocabularyRepository),
14789 entities: Arc::new(NoopEntityRepository),
14790 transformations: Arc::new(NoopTransformationRepository),
14791 tags: Arc::new(NoopTagRepository),
14792 });
14793
14794 let counts = block_on(service.datastore_content_counts()).unwrap();
14795
14796 assert_eq!(counts.studies.count, Some(0));
14797 assert_eq!(counts.assets.count, Some(0));
14798 assert_eq!(counts.datasets.count, Some(0));
14799 assert_eq!(counts.datafiles.count, Some(0));
14800 assert_eq!(counts.entities.count, Some(0));
14801 assert_eq!(counts.variables.count, Some(0));
14802 assert_eq!(counts.vocabularies.count, Some(0));
14803 assert_eq!(counts.transformations.count, Some(0));
14804 }
14805
14806 #[test]
14807 fn datastore_content_counts_reports_populated_summary() {
14808 let asset_repositories = AssetWorkflowRepositories::new();
14809 let entity_repositories = EntityGraphRepositories::new();
14810 let study = test_study("count_summary_study");
14811 let domain = test_domain(80_001, "count_summary_domain");
14812 asset_repositories.seed_study(study.clone());
14813 asset_repositories.seed_study_domain(study.study_id, domain.clone());
14814
14815 let dataset_asset = block_on(
14816 asset_repositories
14817 .assets
14818 .save_asset(&test_asset(study.study_id, "count_dataset_asset")),
14819 )
14820 .unwrap();
14821 let dataset_version = block_on(asset_repositories.assets.save_asset_version(
14822 &test_asset_version(dataset_asset.asset_id, 1, 0, 0, "dataset version"),
14823 ))
14824 .unwrap();
14825 asset_repositories.seed_dataset(DatasetRecord {
14826 dataset_id: dataset_version.version_id,
14827 });
14828
14829 let file_asset = block_on(
14830 asset_repositories
14831 .assets
14832 .save_asset(&test_asset(study.study_id, "count_file_asset")),
14833 )
14834 .unwrap();
14835 let file_version = block_on(asset_repositories.assets.save_asset_version(
14836 &test_asset_version(file_asset.asset_id, 1, 0, 0, "file version"),
14837 ))
14838 .unwrap();
14839 block_on(
14840 asset_repositories
14841 .assets
14842 .save_datafile(&test_datafile(file_version.version_id)),
14843 )
14844 .unwrap();
14845
14846 asset_repositories.seed_variable(test_variable(
14847 90_001,
14848 domain.domain_id.0,
14849 "participant_id",
14850 KeyRole::External,
14851 None,
14852 ));
14853 asset_repositories.seed_variable(test_variable(
14854 90_002,
14855 domain.domain_id.0,
14856 "age_years",
14857 KeyRole::None,
14858 None,
14859 ));
14860 asset_repositories.seed_vocabulary(test_vocabulary(
14861 91_001,
14862 domain.domain_id.0,
14863 "sex_codes",
14864 ));
14865 block_on(
14866 entity_repositories
14867 .entities
14868 .create_entity(&NewEntityRecord {
14869 domain_id: domain.domain_id,
14870 name: nc_name("participant"),
14871 description: None,
14872 agent_instructions: None,
14873 ontology_namespace: None,
14874 ontology_class: None,
14875 }),
14876 )
14877 .unwrap();
14878 block_on(
14879 asset_repositories
14880 .transformations
14881 .create_transformation(&test_transformation(
14882 1,
14883 TransformationType::Ingest,
14884 "ingest",
14885 )),
14886 )
14887 .unwrap();
14888 block_on(
14889 asset_repositories
14890 .transformations
14891 .create_transformation(&test_transformation(
14892 2,
14893 TransformationType::Transform,
14894 "transform",
14895 )),
14896 )
14897 .unwrap();
14898
14899 let service = AppService::new(AppRepositories {
14900 registrations: asset_repositories.domains.clone(),
14901 domains: asset_repositories.domains.clone(),
14902 studies: asset_repositories.studies.clone(),
14903 study_domains: asset_repositories.study_domains.clone(),
14904 study_governance: Arc::new(NoopStudyGovernanceRepository),
14905 assets: asset_repositories.assets.clone(),
14906 variables: asset_repositories.variables.clone(),
14907 vocabularies: asset_repositories.vocabularies.clone(),
14908 entities: entity_repositories.entities.clone(),
14909 transformations: asset_repositories.transformations.clone(),
14910 tags: Arc::new(NoopTagRepository),
14911 });
14912
14913 let counts = block_on(service.datastore_content_counts()).unwrap();
14914
14915 assert_eq!(counts.studies.count, Some(1));
14916 assert_eq!(counts.assets.count, Some(2));
14917 assert_eq!(counts.datasets.count, Some(1));
14918 assert_eq!(counts.datafiles.count, Some(1));
14919 assert_eq!(counts.entities.count, Some(1));
14920 assert_eq!(counts.variables.count, Some(2));
14921 assert_eq!(counts.vocabularies.count, Some(1));
14922 assert_eq!(counts.transformations.count, Some(2));
14923 }
14924
14925 #[test]
14926 fn datastore_content_counts_marks_partially_unsupported_categories() {
14927 let repositories = AssetWorkflowRepositories::new();
14928 let study = test_study("count_partial_study");
14929 let domain = test_domain(80_002, "count_partial_domain");
14930 repositories.seed_study(study.clone());
14931 repositories.seed_study_domain(study.study_id, domain);
14932
14933 let counts = block_on(repositories.service().datastore_content_counts()).unwrap();
14934
14935 assert_eq!(counts.studies.count, Some(1));
14936 assert_eq!(counts.assets.count, Some(0));
14937 assert_eq!(
14938 counts.entities.status,
14939 crate::DatastoreContentCountStatus::Unsupported
14940 );
14941 }
14942
14943 #[test]
14944 fn register_domain_adds_domain_without_creating_study() {
14945 let repositories = RegistrationRepositories::new();
14946 let service = repositories.service();
14947 let domain = test_domain(41, "p1_10_domain_only");
14948
14949 let registered = block_on(service.register_domain(RegisterDomainRequest {
14950 domain: domain.clone(),
14951 }))
14952 .expect("domain registration should succeed");
14953
14954 assert_eq!(registered, domain);
14955 assert!(
14956 block_on(repositories.studies.list_studies())
14957 .expect("study list should read")
14958 .is_empty()
14959 );
14960 assert!(
14961 repositories
14962 .state()
14963 .study_domains
14964 .lock()
14965 .expect("study-domain lock should not be poisoned")
14966 .is_empty()
14967 );
14968 }
14969
14970 #[test]
14971 fn register_new_domain_assigns_domain_id() {
14972 let repositories = RegistrationRepositories::new();
14973 let service = repositories.service();
14974 let domain = test_new_domain("p1_10_new_domain_only");
14975
14976 let registered = block_on(service.register_new_domain(RegisterNewDomainRequest {
14977 domain: domain.clone(),
14978 }))
14979 .expect("new domain registration should succeed");
14980
14981 assert_eq!(registered, domain);
14982 assert_eq!(registered.domain_id, DomainId(1));
14983 assert_eq!(
14984 block_on(repositories.domains.get_domain_by_id(registered.domain_id))
14985 .expect("assigned domain should be readable"),
14986 Some(registered)
14987 );
14988 }
14989
14990 #[test]
14991 fn command_domain_facade_lists_gets_and_adds_domains() {
14992 let repositories = RegistrationRepositories::new();
14993 let service = repositories.service();
14994
14995 let added = block_on(service.add_domain(AddDomainRequest {
14996 name: "cli_domain".to_string(),
14997 description: Some("CLI domain #curated".to_string()),
14998 uri: Some("https://example.test/cli".to_string()),
14999 }))
15000 .expect("domain add facade should create a domain")
15001 .domain;
15002 let repeated = block_on(service.add_domain(AddDomainRequest {
15003 name: "cli_domain".to_string(),
15004 description: Some("CLI domain #curated".to_string()),
15005 uri: Some("https://example.test/cli".to_string()),
15006 }))
15007 .expect("exact Domain repeat should reuse the Domain")
15008 .domain;
15009 let listed = block_on(service.list_domains(ListDomainsRequest {}))
15010 .expect("domain list facade should read domains");
15011 let fetched = block_on(service.get_domain(GetDomainRequest {
15012 domain: DomainSelector::Name {
15013 name: "cli_domain".to_string(),
15014 },
15015 }))
15016 .expect("domain get facade should read by name");
15017
15018 assert_eq!(added.domain_id, DomainId(1));
15019 assert_eq!(repeated, added);
15020 assert_eq!(listed.domains, vec![added.clone()]);
15021 assert_eq!(fetched, Some(added));
15022 assert_eq!(
15023 block_on(repositories.tags.list_tags()).unwrap(),
15024 vec![ahri_tre_types::TagRecord {
15025 tag_id: TagId(1),
15026 name: "curated".to_string(),
15027 }]
15028 );
15029 }
15030
15031 #[test]
15032 fn command_domain_and_study_facades_accept_id_selectors() {
15033 let repositories = RegistrationRepositories::new();
15034 let service = repositories.service();
15035 let domain = test_domain(48, "p1_20i_id_domain");
15036 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
15037 let registration = block_on(service.add_study(AddStudyRequest {
15038 name: "p1_20i_id_study".to_string(),
15039 external_id: "id-study-ext".to_string(),
15040 study_type: "1".to_string(),
15041 domain: DomainSelector::Id {
15042 domain_id: domain.domain_id,
15043 },
15044 description: None,
15045 }))
15046 .expect("study add should resolve domain by id")
15047 .registration;
15048
15049 let fetched_domain = block_on(service.get_domain(GetDomainRequest {
15050 domain: DomainSelector::Id {
15051 domain_id: domain.domain_id,
15052 },
15053 }))
15054 .expect("domain get by id should read")
15055 .expect("domain should exist");
15056 let fetched_study = block_on(service.get_study(GetStudyRequest {
15057 study: StudySelector::Id {
15058 study_id: registration.study.study_id,
15059 },
15060 }))
15061 .expect("study get by id should read")
15062 .expect("study should exist");
15063
15064 assert_eq!(fetched_domain, domain);
15065 assert_eq!(fetched_study.study, registration.study);
15066 assert_eq!(fetched_study.domains, vec![domain]);
15067 }
15068
15069 #[test]
15070 fn study_name_selector_validates_nested_domain_id() {
15071 let repositories = RegistrationRepositories::new();
15072 let service = repositories.service();
15073 let linked_domain = test_domain(49, "p1_20i_linked_domain");
15074 let other_domain = test_domain(50, "p1_20i_other_domain");
15075 block_on(repositories.domains.save_domain(&linked_domain))
15076 .expect("linked domain should save");
15077 block_on(repositories.domains.save_domain(&other_domain))
15078 .expect("other domain should save");
15079 let registration = block_on(service.add_study(AddStudyRequest {
15080 name: "p1_20i_nested_study".to_string(),
15081 external_id: "nested-study-ext".to_string(),
15082 study_type: "1".to_string(),
15083 domain: DomainSelector::Id {
15084 domain_id: linked_domain.domain_id,
15085 },
15086 description: None,
15087 }))
15088 .expect("fixture study should save")
15089 .registration;
15090
15091 let matched = block_on(service.get_study(GetStudyRequest {
15092 study: StudySelector::Name {
15093 domain: Some(DomainSelector::Id {
15094 domain_id: linked_domain.domain_id,
15095 }),
15096 name: registration.study.name.as_str().to_string(),
15097 },
15098 }))
15099 .expect("nested domain id should validate")
15100 .expect("study should exist");
15101 let mismatch = block_on(service.get_study(GetStudyRequest {
15102 study: StudySelector::Name {
15103 domain: Some(DomainSelector::Id {
15104 domain_id: other_domain.domain_id,
15105 }),
15106 name: registration.study.name.as_str().to_string(),
15107 },
15108 }))
15109 .expect_err("mismatched nested domain id should fail");
15110
15111 assert_eq!(matched.study.study_id, registration.study.study_id);
15112 assert!(matches!(mismatch, AppError::Conflict(_)));
15113 }
15114
15115 #[test]
15116 fn command_domain_facade_rejects_invalid_names() {
15117 let service = RegistrationRepositories::new().service();
15118
15119 let error = block_on(service.add_domain(AddDomainRequest {
15120 name: "bad name".to_string(),
15121 description: None,
15122 uri: None,
15123 }))
15124 .expect_err("invalid domain names should be rejected");
15125
15126 assert!(
15127 matches!(error, AppError::Validation(message) if message.contains("domain name is invalid"))
15128 );
15129 }
15130
15131 #[test]
15132 fn governed_access_grant_preserves_qualified_study_scope() {
15133 let repositories = RegistrationRepositories::new();
15134 let mut service = repositories.service();
15135 service.datastore_id = Some(uuid(5));
15136 let linked = test_domain(811, "access_linked");
15137 let other = test_domain(812, "access_other");
15138 block_on(repositories.domains.save_domain(&linked)).unwrap();
15139 block_on(repositories.domains.save_domain(&other)).unwrap();
15140 let registration = block_on(service.add_study(AddStudyRequest {
15141 name: "governed_access".into(),
15142 external_id: "access".into(),
15143 study_type: "1".into(),
15144 domain: DomainSelector::Id {
15145 domain_id: linked.domain_id,
15146 },
15147 description: None,
15148 }))
15149 .unwrap()
15150 .registration;
15151 let before = block_on(
15152 service.list_study_access_grants(ListStudyAccessGrantsRequest {
15153 study_id: registration.study.study_id,
15154 }),
15155 )
15156 .unwrap();
15157 let request = ahri_tre_protocol::study::StudyAccessGrantRequest {
15158 session: None,
15159 study: ahri_tre_protocol::study::StudySelector::Name {
15160 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15161 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15162 name: ahri_tre_protocol::PublicName::new("access_other").unwrap(),
15163 }),
15164 },
15165 target_user: "recipient".into(),
15166 };
15167 assert!(matches!(
15168 block_on(service.grant_governed_study_access(request)),
15169 Err(AppError::Conflict(_))
15170 ));
15171 let after = block_on(
15172 service.list_study_access_grants(ListStudyAccessGrantsRequest {
15173 study_id: registration.study.study_id,
15174 }),
15175 )
15176 .unwrap();
15177 assert_eq!(after, before);
15178 assert!(matches!(
15179 block_on(service.list_governed_study_access(
15180 ahri_tre_protocol::study::StudyAccessListRequest {
15181 session: None,
15182 study: ahri_tre_protocol::study::StudySelector::Name {
15183 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15184 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15185 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15186 }),
15187 },
15188 }
15189 )),
15190 Err(AppError::Conflict(_))
15191 ));
15192 assert!(matches!(
15193 block_on(service.revoke_governed_study_access(
15194 ahri_tre_protocol::study::StudyAccessRevokeRequest {
15195 session: None,
15196 study: ahri_tre_protocol::study::StudySelector::Name {
15197 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15198 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15199 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15200 }),
15201 },
15202 target_user: "recipient".into(),
15203 }
15204 )),
15205 Err(AppError::Conflict(_))
15206 ));
15207 assert!(matches!(
15208 block_on(service.list_governed_study_custodians(
15209 ahri_tre_protocol::study::StudyCustodiansListRequest {
15210 session: None,
15211 study: ahri_tre_protocol::study::StudySelector::Name {
15212 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15213 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15214 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15215 }),
15216 },
15217 }
15218 )),
15219 Err(AppError::Conflict(_))
15220 ));
15221 assert!(matches!(
15222 block_on(service.add_governed_delegate_custodian(
15223 ahri_tre_protocol::study::StudyCustodianMutationRequest {
15224 session: None,
15225 study: ahri_tre_protocol::study::StudySelector::Name {
15226 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15227 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15228 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15229 }),
15230 },
15231 target_user: "recipient".into(),
15232 }
15233 )),
15234 Err(AppError::Conflict(_))
15235 ));
15236 assert!(matches!(
15237 block_on(service.transfer_governed_primary_custodian(
15238 ahri_tre_protocol::study::StudyCustodianMutationRequest {
15239 session: None,
15240 study: ahri_tre_protocol::study::StudySelector::Name {
15241 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15242 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15243 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15244 }),
15245 },
15246 target_user: "recipient".into(),
15247 }
15248 )),
15249 Err(AppError::Conflict(_))
15250 ));
15251 assert!(matches!(
15252 block_on(service.remove_governed_delegate_custodian(
15253 ahri_tre_protocol::study::StudyCustodianMutationRequest {
15254 session: None,
15255 study: ahri_tre_protocol::study::StudySelector::Name {
15256 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15257 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15258 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15259 }),
15260 },
15261 target_user: "recipient".into(),
15262 }
15263 )),
15264 Err(AppError::Conflict(_))
15265 ));
15266 assert!(matches!(
15267 block_on(service.list_governed_study_duo(
15268 ahri_tre_protocol::study::StudyDuoListRequest {
15269 session: None,
15270 study: ahri_tre_protocol::study::StudySelector::Name {
15271 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15272 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15273 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15274 }),
15275 },
15276 }
15277 )),
15278 Err(AppError::Conflict(_))
15279 ));
15280 assert!(matches!(
15281 block_on(service.replace_governed_study_duo(
15282 ahri_tre_protocol::study::StudyDuoReplaceRequest {
15283 session: None,
15284 study: ahri_tre_protocol::study::StudySelector::Name {
15285 name: ahri_tre_protocol::PublicName::new("governed_access").unwrap(),
15286 domain: Some(ahri_tre_protocol::domain::DomainSelector::Name {
15287 name: ahri_tre_protocol::PublicName::new("access_other").unwrap()
15288 }),
15289 },
15290 restrictions: vec![],
15291 }
15292 )),
15293 Err(AppError::Conflict(_))
15294 ));
15295 }
15296
15297 #[test]
15298 fn governed_study_domain_rejects_foreign_origin_before_linking() {
15299 use ahri_tre_protocol::{PublicUuid, refs::ObjectKind};
15300 let repositories = RegistrationRepositories::new();
15301 let mut service = repositories.service();
15302 service.datastore_id = Some(uuid(5));
15303 let linked = test_domain(801, "linked");
15304 let additional = test_domain(802, "additional");
15305 block_on(repositories.domains.save_domain(&linked)).unwrap();
15306 block_on(repositories.domains.save_domain(&additional)).unwrap();
15307 let registration = block_on(service.add_study(AddStudyRequest {
15308 name: "governed_study".into(),
15309 external_id: "governed-study".into(),
15310 study_type: "1".into(),
15311 domain: DomainSelector::Id {
15312 domain_id: linked.domain_id,
15313 },
15314 description: None,
15315 }))
15316 .unwrap()
15317 .registration;
15318 let request = ahri_tre_protocol::study::AddStudyDomainRequest {
15319 session: None,
15320 study: ahri_tre_protocol::study::StudySelector::Id {
15321 study: crate::projections::object_ref(
15322 PublicUuid::from_uuid(uuid(5)),
15323 ObjectKind::Study,
15324 registration.study.study_id.0,
15325 ),
15326 },
15327 domain: ahri_tre_protocol::domain::DomainSelector::Id {
15328 domain: crate::projections::domain_ref(
15329 PublicUuid::from_uuid(uuid(6)),
15330 additional.domain_id,
15331 ),
15332 },
15333 };
15334 assert!(matches!(
15335 block_on(service.add_governed_study_domain(request)),
15336 Err(AppError::Conflict(_))
15337 ));
15338 let after = block_on(service.get_study(GetStudyRequest {
15339 study: StudySelector::Id {
15340 study_id: registration.study.study_id,
15341 },
15342 }))
15343 .unwrap()
15344 .unwrap();
15345 assert_eq!(after.domains, vec![linked]);
15346 }
15347
15348 #[test]
15349 fn governed_semantic_delete_resolves_canonical_identity_and_domain_constraint() {
15350 use ahri_tre_protocol::{PublicUuid, lifecycle::*, refs::ObjectKind};
15351 let repositories = EntityGraphRepositories::new();
15352 let mut service = repositories.service();
15353 service.datastore_id = Some(uuid(5));
15354 let domain = test_domain(891, "delete_domain");
15355 let other = test_domain(892, "other_delete_domain");
15356 repositories.seed_domain(domain.clone());
15357 repositories.seed_domain(other.clone());
15358 let entity = block_on(service.add_entity(AddEntityRequest {
15359 domain: DomainSelector::Id {
15360 domain_id: domain.domain_id,
15361 },
15362 name: "delete_entity".into(),
15363 description: None,
15364 agent_instructions: None,
15365 ontology_namespace: None,
15366 ontology_class: None,
15367 }))
15368 .unwrap();
15369 let reference = ahri_tre_protocol::refs::ObjectRef {
15370 datastore_id: PublicUuid::from_uuid(uuid(5)),
15371 kind: ObjectKind::Entity,
15372 id: ahri_tre_protocol::refs::encode_scoped_integer_ref("entity", entity.entity_id.0),
15373 };
15374 let request = |target| SemanticDeleteRequest {
15375 session: None,
15376 target,
15377 intent: DestructiveIntent {
15378 reason: "retire unused definition".into(),
15379 actor: None,
15380 dry_run: true,
15381 confirmed: false,
15382 },
15383 };
15384 let by_ref = block_on(service.plan_governed_semantic_delete(
15385 request(SemanticDeleteSelector::Reference {
15386 reference,
15387 domain: None,
15388 }),
15389 "owner",
15390 ))
15391 .unwrap()
15392 .plan
15393 .unwrap();
15394 let by_name = block_on(service.plan_governed_semantic_delete(
15395 request(SemanticDeleteSelector::Entity {
15396 entity: ahri_tre_protocol::model::EntitySelector::Name {
15397 domain: ahri_tre_protocol::domain::DomainSelector::Name {
15398 name: ahri_tre_protocol::PublicName::new("delete_domain").unwrap(),
15399 },
15400 name: ahri_tre_protocol::PublicName::new("delete_entity").unwrap(),
15401 },
15402 }),
15403 "owner",
15404 ))
15405 .unwrap()
15406 .plan
15407 .unwrap();
15408 assert_eq!(by_ref.target_identity, by_name.target_identity);
15409 assert!(matches!(
15410 by_name.target,
15411 SemanticDeleteTarget::Entity {
15412 entity: EntitySelector::Id { .. }
15413 }
15414 ));
15415 let mut other = other;
15417 other.name = domain.name.clone();
15418 repositories.seed_domain(other.clone());
15419 assert!(matches!(
15420 block_on(service.plan_governed_semantic_delete(
15421 request(SemanticDeleteSelector::Reference {
15422 reference,
15423 domain: Some(ahri_tre_protocol::domain::DomainSelector::Id {
15424 domain: crate::projections::domain_ref(
15425 PublicUuid::from_uuid(uuid(5)),
15426 other.domain_id
15427 )
15428 }),
15429 }),
15430 "owner"
15431 )),
15432 Err(AppError::Conflict(_))
15433 ));
15434 }
15435
15436 #[test]
15437 fn governed_withdrawal_requires_concrete_identity_and_preserves_pinned_scope() {
15438 use ahri_tre_protocol::{PublicUuid, lifecycle::*, refs::ObjectKind};
15439 let repositories = AssetWorkflowRepositories::new();
15440 let mut service = repositories.service();
15441 service.datastore_id = Some(uuid(5));
15442 let study = test_study("withdraw_scope");
15443 repositories.seed_study(study.clone());
15444 repositories.grant_study_custodianship(study.study_id, "owner");
15445 let asset = test_asset(study.study_id, "withdraw_dataset");
15446 let first =
15447 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, false, "historical");
15448 seed_completed_dataset_version(&repositories, &asset, 2, 0, 0, true, "current");
15449 let request = DatasetWithdrawRequest {
15450 session: None,
15451 dataset: ahri_tre_protocol::dataset::DatasetSelector::Id {
15452 dataset: crate::projections::object_ref(
15453 PublicUuid::from_uuid(uuid(5)),
15454 ObjectKind::AssetVersion,
15455 first.version_id.0,
15456 ),
15457 study: None,
15458 },
15459 version: None,
15460 intent: DestructiveIntent {
15461 reason: "withdraw".into(),
15462 actor: None,
15463 dry_run: true,
15464 confirmed: true,
15465 },
15466 force: true,
15467 drop_lake_table: false,
15468 };
15469 let plan = block_on(service.plan_governed_withdrawal(&request, "owner")).unwrap();
15470 assert_eq!(plan.version, "1.0.0");
15471 let mut latest = request.clone();
15472 latest.version = Some("latest".into());
15473 assert!(matches!(
15474 block_on(service.plan_governed_withdrawal(&latest, "owner")),
15475 Err(AppError::Validation(_))
15476 ));
15477 let mut conflict = request.clone();
15478 conflict.version = Some("2.0.0".into());
15479 assert!(matches!(
15480 block_on(service.plan_governed_withdrawal(&conflict, "owner")),
15481 Err(AppError::Conflict(_))
15482 ));
15483 assert!(block_on(service.plan_governed_withdrawal(&request, "wrong-user")).is_err());
15484 for version in ["1.0.0", "2.0.0"] {
15485 block_on(service.withdraw_dataset_version(
15486 None,
15487 WithdrawDatasetVersionRequest {
15488 study_id: study.study_id,
15489 dataset_name: asset.name.clone(),
15490 version: version.into(),
15491 reason: "retire final history".into(),
15492 actor: Some("owner".into()),
15493 force: true,
15494 drop_lake_table: false,
15495 },
15496 ))
15497 .unwrap();
15498 }
15499 let retry = block_on(service.plan_governed_withdrawal(&request, "owner"));
15500 assert!(
15501 retry.is_ok(),
15502 "pinned withdrawn history remains addressable: {retry:?}"
15503 );
15504 }
15505
15506 #[test]
15507 fn governed_delete_planning_preserves_version_and_whole_asset_intent() {
15508 use ahri_tre_protocol::{PublicUuid, lifecycle::*, refs::ObjectKind};
15509 let repositories = AssetWorkflowRepositories::new();
15510 let mut service = repositories.service();
15511 service.datastore_id = Some(uuid(5));
15512 let study = test_study("delete_scope");
15513 repositories.seed_study(study.clone());
15514 repositories.grant_study_custodianship(study.study_id, "owner");
15515 let mut asset = test_asset(study.study_id, "delete_file");
15516 asset.asset_type = AssetType::File;
15517 block_on(repositories.assets.save_asset(&asset)).unwrap();
15518 let version = test_asset_version(asset.asset_id, 1, 0, 0, "initial");
15519 block_on(repositories.assets.save_asset_version(&version)).unwrap();
15520 let request = DataFileDeleteRequest {
15521 session: None,
15522 datafile: ahri_tre_protocol::datafile::DataFileSelector::Id {
15523 datafile: crate::projections::object_ref(
15524 PublicUuid::from_uuid(uuid(5)),
15525 ObjectKind::AssetVersion,
15526 version.version_id.0,
15527 ),
15528 study: None,
15529 },
15530 preconditions: DeletePreconditions {
15531 version: None,
15532 all_versions: false,
15533 },
15534 intent: DestructiveIntent {
15535 reason: "retire".into(),
15536 actor: None,
15537 dry_run: true,
15538 confirmed: false,
15539 },
15540 policy: ArchiveDeletePolicy {
15541 archive: true,
15542 cascade: true,
15543 force: true,
15544 },
15545 };
15546 let plan = block_on(
15547 service
15548 .plan_governed_delete(&GovernedDeleteRequest::DataFile(request.clone()), "owner"),
15549 )
15550 .unwrap();
15551 assert_eq!(
15552 plan.planning.plan.unwrap().resolved_version,
15553 Some(version.version_id)
15554 );
15555 for dry_run in [true, false] {
15556 let mut conflict = request.clone();
15557 conflict.intent.dry_run = dry_run;
15558 conflict.preconditions.all_versions = true;
15559 assert!(matches!(
15560 block_on(
15561 service
15562 .plan_governed_delete(&GovernedDeleteRequest::DataFile(conflict), "owner")
15563 ),
15564 Err(AppError::Conflict(_))
15565 ));
15566 }
15567 let mut spoofed = request;
15568 spoofed.intent.actor = Some("different-user".into());
15569 assert!(matches!(
15570 block_on(
15571 service.plan_governed_delete(&GovernedDeleteRequest::DataFile(spoofed), "owner")
15572 ),
15573 Err(AppError::Validation(_))
15574 ));
15575 }
15576
15577 #[test]
15578 fn governed_asset_duo_pins_versions_and_rejects_conflicting_mutation() {
15579 use ahri_tre_protocol::{PublicUuid, asset::AssetSelector, refs::ObjectKind};
15580 let repositories = AssetWorkflowRepositories::new();
15581 let mut service = repositories.service();
15582 service.datastore_id = Some(uuid(5));
15583 let study = test_study("governed_duo");
15584 repositories.seed_study(study.clone());
15585 let asset = test_asset(study.study_id, "governed_duo_asset");
15586 block_on(repositories.assets.save_asset(&asset)).unwrap();
15587 let mut first = test_asset_version(asset.asset_id, 1, 0, 0, "first");
15588 first.is_latest = Some(false);
15589 let mut second = test_asset_version(asset.asset_id, 1, 0, 1, "second");
15590 second.is_latest = Some(true);
15591 block_on(repositories.assets.save_asset_version(&first)).unwrap();
15592 block_on(repositories.assets.save_asset_version(&second)).unwrap();
15593 let pinned = AssetSelector::Id {
15594 asset: crate::projections::object_ref(
15595 PublicUuid::from_uuid(uuid(5)),
15596 ObjectKind::AssetVersion,
15597 first.version_id.0,
15598 ),
15599 study: None,
15600 asset_type: None,
15601 };
15602 let replace = ahri_tre_protocol::asset::AssetDuoReplaceRequest {
15603 session: None,
15604 asset: pinned.clone(),
15605 version: None,
15606 restrictions: vec![ahri_tre_protocol::asset::AssetDuoRestrictionInput {
15607 code: Some("DUO:0000007".into()),
15608 duo_id: None,
15609 qualifier_ontology_namespace: None,
15610 qualifier_ontology_id: None,
15611 qualifier_text: None,
15612 qualifier_date: None,
15613 qualifier_integer: None,
15614 }],
15615 };
15616 block_on(service.replace_governed_asset_duo(replace.clone())).unwrap();
15617 assert!(matches!(
15618 block_on(service.replace_governed_asset_duo(
15619 ahri_tre_protocol::asset::AssetDuoReplaceRequest {
15620 version: Some("latest".into()),
15621 ..replace
15622 }
15623 )),
15624 Err(AppError::Conflict(_))
15625 ));
15626 let result = block_on(service.list_governed_asset_duo(
15627 ahri_tre_protocol::asset::AssetDuoListRequest {
15628 session: None,
15629 asset: pinned.clone(),
15630 version: None,
15631 effective: true,
15632 },
15633 ))
15634 .unwrap();
15635 assert_eq!(result.version.version_id, first.version_id);
15636 assert_eq!(result.restrictions[0].duo_id, "DUO:0000007");
15637 let denied = block_on(service.clear_governed_asset_duo(
15638 ahri_tre_protocol::asset::AssetDuoClearRequest {
15639 session: None,
15640 asset: pinned,
15641 version: Some("latest".into()),
15642 },
15643 ));
15644 assert!(matches!(denied, Err(AppError::Conflict(_))));
15645 let latest = block_on(service.list_governed_asset_duo(
15646 ahri_tre_protocol::asset::AssetDuoListRequest {
15647 session: None,
15648 asset: AssetSelector::Name {
15649 study: ahri_tre_protocol::study::StudySelector::Name {
15650 domain: None,
15651 name: ahri_tre_protocol::PublicName::new("governed_duo").unwrap(),
15652 },
15653 name: ahri_tre_protocol::PublicName::new("governed_duo_asset").unwrap(),
15654 asset_type: None,
15655 },
15656 version: None,
15657 effective: false,
15658 },
15659 ))
15660 .unwrap();
15661 assert_eq!(latest.version.version_id, second.version_id);
15662 }
15663
15664 #[test]
15665 fn catalogue_datafile_references_preserve_origin_and_pin_versions() {
15666 use ahri_tre_protocol::{
15667 PublicUuid,
15668 datafile::DataFileSelector,
15669 refs::{ObjectKind, ObjectRef},
15670 };
15671 let repositories = AssetWorkflowRepositories::new();
15672 let mut service = repositories.service();
15673 service.datastore_id = Some(uuid(5));
15674 let study = test_study("catalogue_study");
15675 repositories.seed_study(study.clone());
15676 let mut asset = test_asset(study.study_id, "catalogue_file");
15677 asset.asset_type = AssetType::File;
15678 block_on(repositories.assets.save_asset(&asset)).unwrap();
15679 let mut first = test_asset_version(asset.asset_id, 1, 0, 0, "first");
15680 first.is_latest = Some(false);
15681 let mut second = test_asset_version(asset.asset_id, 1, 0, 1, "second");
15682 second.is_latest = Some(true);
15683 for version in [&first, &second] {
15684 block_on(repositories.assets.save_asset_version(version)).unwrap();
15685 block_on(
15686 repositories
15687 .assets
15688 .save_datafile(&test_datafile(version.version_id)),
15689 )
15690 .unwrap();
15691 }
15692 let logical = ObjectRef {
15693 datastore_id: PublicUuid::from_uuid(uuid(5)),
15694 kind: ObjectKind::Asset,
15695 id: PublicUuid::from_uuid(asset.asset_id.0),
15696 };
15697 let pinned = ObjectRef {
15698 kind: ObjectKind::AssetVersion,
15699 id: PublicUuid::from_uuid(first.version_id.0),
15700 ..logical
15701 };
15702 let read = |reference, version| {
15703 block_on(service.read_catalogue_datafile(
15704 DataFileSelector::Id {
15705 study: None,
15706 datafile: reference,
15707 },
15708 version,
15709 ))
15710 };
15711 assert_eq!(
15712 read(logical, Some(" v1.0.0 "))
15713 .unwrap()
15714 .1
15715 .version
15716 .version_id,
15717 first.version_id
15718 );
15719 assert_eq!(
15720 read(logical, Some("LATEST")).unwrap().1.version.version_id,
15721 second.version_id
15722 );
15723 assert_eq!(
15724 read(pinned, None).unwrap().1.version.version_id,
15725 first.version_id
15726 );
15727 assert!(matches!(
15728 read(pinned, Some("latest")),
15729 Err(AppError::Conflict(_))
15730 ));
15731 let foreign = ObjectRef {
15732 datastore_id: PublicUuid::from_uuid(uuid(6)),
15733 ..logical
15734 };
15735 assert!(matches!(read(foreign, None), Err(AppError::Conflict(_))));
15736 let wrong_kind = ObjectRef {
15737 kind: ObjectKind::Study,
15738 ..logical
15739 };
15740 assert!(matches!(
15741 read(wrong_kind, None),
15742 Err(AppError::Validation(_))
15743 ));
15744 }
15745
15746 #[test]
15747 fn catalogue_dataset_selects_historical_versions_and_checks_repeated_scope() {
15748 use ahri_tre_protocol::{
15749 PublicUuid,
15750 dataset::DatasetSelector,
15751 refs::{ObjectKind, ObjectRef},
15752 };
15753 let repositories = AssetWorkflowRepositories::new();
15754 let mut service = repositories.service();
15755 service.datastore_id = Some(uuid(5));
15756 let study = test_study("dataset_scope");
15757 let other = test_study("other_scope");
15758 repositories.seed_study(study.clone());
15759 repositories.seed_study(other.clone());
15760 let asset = test_asset(study.study_id, "dataset");
15761 let first =
15762 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, false, "historical");
15763 let latest =
15764 seed_completed_dataset_version(&repositories, &asset, 2, 0, 0, true, "current");
15765 let logical = ObjectRef {
15766 datastore_id: PublicUuid::from_uuid(uuid(5)),
15767 kind: ObjectKind::Asset,
15768 id: PublicUuid::from_uuid(asset.asset_id.0),
15769 };
15770 let pinned = ObjectRef {
15771 kind: ObjectKind::AssetVersion,
15772 id: PublicUuid::from_uuid(first.version_id.0),
15773 ..logical
15774 };
15775 let read = |dataset, study| {
15776 block_on(service.read_catalogue_dataset(DatasetSelector::Id { dataset, study }, false))
15777 };
15778 assert_eq!(
15779 read(logical, None).unwrap().metadata.dataset.dataset_id,
15780 latest.version_id
15781 );
15782 assert_eq!(
15783 read(pinned, None).unwrap().metadata.dataset.dataset_id,
15784 first.version_id
15785 );
15786 let scope = |name: &str| ahri_tre_protocol::study::StudySelector::Name {
15787 domain: None,
15788 name: ahri_tre_protocol::PublicName::new(name).unwrap(),
15789 };
15790 assert_eq!(
15791 read(pinned, Some(scope(study.name.as_str())))
15792 .unwrap()
15793 .metadata
15794 .dataset
15795 .dataset_id,
15796 first.version_id
15797 );
15798 assert!(matches!(
15799 read(pinned, Some(scope(other.name.as_str()))),
15800 Err(AppError::Conflict(_))
15801 ));
15802 assert!(matches!(
15803 read(
15804 ObjectRef {
15805 id: PublicUuid::from_uuid(uuid(999)),
15806 ..pinned
15807 },
15808 None
15809 ),
15810 Err(AppError::NotFound(_))
15811 ));
15812 let mut second_session = repositories.service();
15814 second_session.datastore_id = service.datastore_id;
15815 assert_eq!(
15816 block_on(second_session.read_catalogue_dataset(
15817 DatasetSelector::Id {
15818 dataset: pinned,
15819 study: None
15820 },
15821 false
15822 ))
15823 .unwrap()
15824 .metadata
15825 .dataset
15826 .dataset_id,
15827 first.version_id
15828 );
15829 }
15830
15831 #[test]
15832 fn catalogue_search_preserves_unmatched_scopes_and_any_of_results() {
15833 use ahri_tre_protocol::{
15834 PublicName,
15835 domain::DomainSelector,
15836 pagination::PageRequest,
15837 search::{PredicateSet, TextMode, TextPredicate},
15838 study::{SearchStudiesRequest, StudyPredicate, StudyTextField},
15839 };
15840 let repositories = AssetWorkflowRepositories::new();
15841 let mut service = repositories.service();
15842 service.datastore_id = Some(uuid(5));
15843 let study = test_study("visible_search");
15844 repositories.seed_study(study.clone());
15845 let missing = StudyPredicate::Domain {
15846 domain: DomainSelector::Name {
15847 name: PublicName::new("absent").unwrap(),
15848 },
15849 };
15850 let matching = StudyPredicate::Text(TextPredicate {
15851 field: StudyTextField::Name,
15852 mode: TextMode::Exact,
15853 value: study.name.as_str().into(),
15854 });
15855 for (predicates, count) in [
15856 (PredicateSet::AllOf(vec![missing.clone()]), 0),
15857 (
15858 PredicateSet::AllOf(vec![missing.clone(), matching.clone()]),
15859 0,
15860 ),
15861 (PredicateSet::AnyOf(vec![missing.clone()]), 0),
15862 (PredicateSet::AnyOf(vec![missing, matching]), 1),
15863 ] {
15864 let (results, cursor) =
15865 block_on(service.search_catalogue_studies(SearchStudiesRequest {
15866 session: None,
15867 predicates,
15868 page: PageRequest::default(),
15869 }))
15870 .unwrap();
15871 assert_eq!(results.len(), count);
15872 assert!(cursor.is_none());
15873 }
15874 }
15875
15876 #[test]
15877 fn catalogue_search_checks_origin_and_preserves_nested_domain_constraints() {
15878 use ahri_tre_protocol::{
15879 PublicUuid,
15880 datafile::{DataFilePredicate, SearchDataFilesRequest},
15881 domain::DomainSelector,
15882 pagination::PageRequest,
15883 refs::{ObjectKind, ObjectRef},
15884 search::PredicateSet,
15885 study::StudySelector,
15886 };
15887 let repositories = AssetWorkflowRepositories::new();
15888 let mut service = repositories.service();
15889 service.datastore_id = Some(uuid(5));
15890 let study = test_study("search_scope");
15891 let linked = test_domain(801, "linked");
15892 let other = test_domain(802, "other");
15893 repositories.seed_study(study.clone());
15894 repositories.seed_study_domain(study.study_id, linked.clone());
15895 repositories.seed_study_domain(test_study("unrelated").study_id, other.clone());
15896 let query = |study| SearchDataFilesRequest {
15897 session: None,
15898 predicates: PredicateSet::AllOf(vec![DataFilePredicate::Study { study }]),
15899 page: PageRequest {
15900 limit: 10,
15901 cursor: None,
15902 },
15903 };
15904 let scoped = |domain| StudySelector::Name {
15905 domain: Some(DomainSelector::Id { domain }),
15906 name: ahri_tre_protocol::PublicName::new(study.name.as_str()).unwrap(),
15907 };
15908 let reference = |id| ObjectRef {
15909 datastore_id: PublicUuid::from_uuid(uuid(5)),
15910 kind: ObjectKind::Domain,
15911 id: ahri_tre_protocol::refs::encode_scoped_integer_ref("domain", id),
15912 };
15913 assert!(matches!(
15914 block_on(
15915 service.search_catalogue_datafiles(query(scoped(reference(other.domain_id.0))))
15916 ),
15917 Err(AppError::Conflict(_))
15918 ));
15919 let foreign = ObjectRef {
15920 datastore_id: PublicUuid::from_uuid(uuid(6)),
15921 ..reference(linked.domain_id.0)
15922 };
15923 assert!(matches!(
15924 block_on(service.search_catalogue_datafiles(query(scoped(foreign)))),
15925 Err(AppError::Conflict(_))
15926 ));
15927 }
15928
15929 #[test]
15930 fn command_domain_facade_creates_distinct_uri_pairs_and_requires_unambiguous_reads() {
15931 let service = RegistrationRepositories::new().service();
15932 let create = |uri: Option<&str>| AddDomainRequest {
15933 name: "shared_domain".to_string(),
15934 description: Some("Catalogue".to_string()),
15935 uri: uri.map(str::to_owned),
15936 };
15937 let first = block_on(service.add_domain(create(Some("https://example.test/first"))))
15938 .unwrap()
15939 .domain;
15940 let second = block_on(service.add_domain(create(Some("https://example.test/second"))))
15941 .expect("an explicit distinct URI creates a distinct Domain")
15942 .domain;
15943 assert_ne!(first.domain_id, second.domain_id);
15944 assert_eq!(
15945 block_on(service.add_domain(create(Some("https://example.test/second"))))
15946 .unwrap()
15947 .domain,
15948 second
15949 );
15950 assert!(matches!(
15951 block_on(service.get_domain(GetDomainRequest {
15952 domain: DomainSelector::Name {
15953 name: "shared_domain".into()
15954 },
15955 })),
15956 Err(AppError::Conflict(_))
15957 ));
15958 assert_eq!(
15959 block_on(service.get_domain(GetDomainRequest {
15960 domain: DomainSelector::Id {
15961 domain_id: first.domain_id
15962 },
15963 }))
15964 .unwrap(),
15965 Some(first.clone())
15966 );
15967 assert!(matches!(
15968 block_on(service.add_domain(create(None))),
15969 Err(AppError::Conflict(_))
15970 ));
15971 assert_eq!(
15972 block_on(service.list_domains(ListDomainsRequest {}))
15973 .unwrap()
15974 .domains
15975 .len(),
15976 2
15977 );
15978 let mut changed = create(Some("https://example.test/first"));
15979 changed.description = Some("Changed".into());
15980 assert!(matches!(
15981 block_on(service.add_domain(changed)),
15982 Err(AppError::Conflict(_))
15983 ));
15984 }
15985
15986 #[test]
15987 fn failed_domain_registration_leaves_no_domain_or_tags() {
15988 let repositories = RegistrationRepositories::new();
15989 let service = repositories.service();
15990 repositories.fail_next_registration_at(RegistrationFailurePoint::DomainTags);
15991
15992 let error = block_on(service.add_domain(AddDomainRequest {
15993 name: "rolled_back_domain".to_string(),
15994 description: Some("Rollback fixture #failure".to_string()),
15995 uri: None,
15996 }))
15997 .expect_err("tag failure must fail Domain registration");
15998
15999 assert!(matches!(error, AppError::Infrastructure(_)));
16000 assert!(
16001 block_on(repositories.domains.list_domains())
16002 .unwrap()
16003 .is_empty()
16004 );
16005 assert!(block_on(repositories.tags.list_tags()).unwrap().is_empty());
16006 }
16007
16008 #[test]
16009 fn failed_explicit_domain_registration_uses_atomic_port() {
16010 let repositories = RegistrationRepositories::new();
16011 let service = repositories.service();
16012 repositories.fail_next_registration_at(RegistrationFailurePoint::DomainTags);
16013 let mut domain = test_domain(150, "rolled_back_explicit_domain");
16014 domain.description = Some("Rollback fixture #failure".to_string());
16015
16016 block_on(service.register_domain(RegisterDomainRequest { domain }))
16017 .expect_err("tag failure must fail explicit Domain registration");
16018
16019 assert!(
16020 block_on(repositories.domains.list_domains())
16021 .unwrap()
16022 .is_empty()
16023 );
16024 assert!(block_on(repositories.tags.list_tags()).unwrap().is_empty());
16025 }
16026
16027 #[test]
16028 fn register_study_requires_at_least_one_domain() {
16029 let service = RegistrationRepositories::new().service();
16030
16031 let error = block_on(service.register_study(RegisterStudyRequest {
16032 study: test_study("p1_10_orphan_study"),
16033 domain_ids: Vec::new(),
16034 }))
16035 .expect_err("study without a domain should fail");
16036
16037 match error {
16038 AppError::Validation(message) => assert!(message.contains("at least one domain")),
16039 other => panic!("expected validation error, got {other:?}"),
16040 }
16041 }
16042
16043 #[test]
16044 fn register_study_rejects_missing_domain() {
16045 let service = RegistrationRepositories::new().service();
16046
16047 let error = block_on(service.register_study(RegisterStudyRequest {
16048 study: test_study("p1_10_missing_domain_study"),
16049 domain_ids: vec![DomainId(404)],
16050 }))
16051 .expect_err("study with a missing domain should fail");
16052
16053 match error {
16054 AppError::NotFound(message) => assert!(message.contains("study domain not found")),
16055 other => panic!("expected not-found error, got {other:?}"),
16056 }
16057 }
16058
16059 #[test]
16060 fn register_new_study_assigns_study_id_and_links_domains() {
16061 let repositories = RegistrationRepositories::new();
16062 let service = repositories.service();
16063 let domain = test_domain(44, "p1_10_new_study_domain");
16064 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16065 let study = test_new_study("p1_10_new_study");
16066
16067 let registration = block_on(service.register_new_study(RegisterNewStudyRequest {
16068 study: study.clone(),
16069 domain_ids: vec![domain.domain_id],
16070 }))
16071 .expect("new study registration should succeed");
16072
16073 assert_eq!(registration.study, study);
16074 assert_ne!(registration.study.study_id.0, uuid::Uuid::nil());
16075 assert_eq!(registration.domains, vec![domain]);
16076 assert_creator_seeded_as_access_grant_and_primary_custodian(
16077 &repositories,
16078 registration.study.study_id,
16079 registration.study.created_by.as_deref().unwrap(),
16080 );
16081 }
16082
16083 #[test]
16084 fn add_study_facade_resolves_domain_name_and_registers_study() {
16085 let repositories = RegistrationRepositories::new();
16086 let service = repositories.service();
16087 let domain = test_domain(46, "p1_10_cli_domain");
16088 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16089
16090 let registration = block_on(service.add_study(AddStudyRequest {
16091 name: "p1_10_cli_study".to_string(),
16092 external_id: "cli-study-ext".to_string(),
16093 study_type: "3".to_string(),
16094 domain: DomainSelector::Name {
16095 name: "p1_10_cli_domain".to_string(),
16096 },
16097 description: Some("CLI study".to_string()),
16098 }))
16099 .expect("CLI-shaped study add should succeed")
16100 .registration;
16101
16102 assert_eq!(registration.study.name.as_str(), "p1_10_cli_study");
16103 assert_eq!(
16104 registration.study.external_id.as_deref(),
16105 Some("cli-study-ext")
16106 );
16107 assert_eq!(
16108 registration.study.study_type_id,
16109 Some(ahri_tre_types::StudyTypeId(3))
16110 );
16111 assert_eq!(registration.domains, vec![domain]);
16112 assert_eq!(
16113 registration.study.created_by.as_deref(),
16114 Some("test_creator")
16115 );
16116 assert_creator_seeded_as_access_grant_and_primary_custodian(
16117 &repositories,
16118 registration.study.study_id,
16119 "test_creator",
16120 );
16121 }
16122
16123 #[test]
16124 fn add_study_facade_accepts_seeded_study_type_names() {
16125 let repositories = RegistrationRepositories::new();
16126 let service = repositories.service();
16127 let domain = test_domain(146, "p1_10_named_type_domain");
16128 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16129
16130 let hdss = block_on(service.add_study(AddStudyRequest {
16131 name: "p1_10_named_hdss_study".to_string(),
16132 external_id: "named-hdss-study-ext".to_string(),
16133 study_type: "HDSS".to_string(),
16134 domain: DomainSelector::Name {
16135 name: "p1_10_named_type_domain".to_string(),
16136 },
16137 description: None,
16138 }))
16139 .expect("HDSS study type name should normalize")
16140 .registration;
16141 assert_eq!(
16142 hdss.study.study_type_id,
16143 Some(ahri_tre_types::StudyTypeId(1))
16144 );
16145
16146 let mixed_methods = block_on(service.add_study(AddStudyRequest {
16147 name: "p1_10_named_mixed_methods_study".to_string(),
16148 external_id: "named-mixed-methods-study-ext".to_string(),
16149 study_type: "mixed methods".to_string(),
16150 domain: DomainSelector::Name {
16151 name: "p1_10_named_type_domain".to_string(),
16152 },
16153 description: None,
16154 }))
16155 .expect("spaced study type name should normalize")
16156 .registration;
16157 assert_eq!(
16158 mixed_methods.study.study_type_id,
16159 Some(ahri_tre_types::StudyTypeId(16))
16160 );
16161 }
16162
16163 #[test]
16164 fn add_study_facade_reuses_existing_study_by_name() {
16165 let repositories = RegistrationRepositories::new();
16166 let service = repositories.service();
16167 let domain = test_domain(147, "p1_10_idempotent_study_domain");
16168 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16169
16170 let first = block_on(service.add_study(AddStudyRequest {
16171 name: "p1_10_idempotent_study".to_string(),
16172 external_id: "idempotent-study-ext".to_string(),
16173 study_type: "HDSS".to_string(),
16174 domain: DomainSelector::Name {
16175 name: domain.name.as_str().to_string(),
16176 },
16177 description: Some("First add #curated".to_string()),
16178 }))
16179 .expect("first study add should create")
16180 .registration;
16181 let second = block_on(service.add_study(AddStudyRequest {
16182 name: "p1_10_idempotent_study".to_string(),
16183 external_id: "idempotent-study-ext".to_string(),
16184 study_type: "HDSS".to_string(),
16185 domain: DomainSelector::Name {
16186 name: domain.name.as_str().to_string(),
16187 },
16188 description: Some("First add #curated".to_string()),
16189 }))
16190 .expect("second study add should reuse the existing study")
16191 .registration;
16192
16193 assert_eq!(second.study.study_id, first.study.study_id);
16194 assert_eq!(second.study.name, first.study.name);
16195 assert_eq!(second.domains, vec![domain]);
16196 assert_eq!(second.tags.len(), 1);
16197 assert_eq!(second.tags[0].name, "curated");
16198 }
16199
16200 #[test]
16201 fn add_study_facade_rejects_conflicting_identity_without_partial_creation() {
16202 let repositories = RegistrationRepositories::new();
16203 let service = repositories.service();
16204 let domain = test_domain(148, "p1_10_conflicting_study_domain");
16205 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16206 let first = block_on(service.add_study(AddStudyRequest {
16207 name: "p1_10_conflicting_study".to_string(),
16208 external_id: "original-external-id".to_string(),
16209 study_type: "HDSS".to_string(),
16210 domain: DomainSelector::Name {
16211 name: domain.name.as_str().to_string(),
16212 },
16213 description: Some("Original".to_string()),
16214 }))
16215 .expect("first Study registration should succeed")
16216 .registration;
16217
16218 let error = block_on(service.add_study(AddStudyRequest {
16219 name: "p1_10_conflicting_study".to_string(),
16220 external_id: "conflicting-external-id".to_string(),
16221 study_type: "HDSS".to_string(),
16222 domain: DomainSelector::Name {
16223 name: domain.name.as_str().to_string(),
16224 },
16225 description: Some("Conflicting".to_string()),
16226 }))
16227 .expect_err("same Study name with a different identity must conflict");
16228
16229 assert!(matches!(error, AppError::Conflict(message) if message.contains("conflicts")));
16230 assert_eq!(
16231 block_on(repositories.studies.list_studies()).unwrap(),
16232 vec![first.study]
16233 );
16234 }
16235
16236 #[test]
16237 fn failed_study_registration_leaves_no_catalogue_or_governance_state() {
16238 for point in [
16239 RegistrationFailurePoint::StudyTags,
16240 RegistrationFailurePoint::StudyDomain,
16241 RegistrationFailurePoint::StudyAccess,
16242 RegistrationFailurePoint::StudyCustodian,
16243 ] {
16244 let repositories = RegistrationRepositories::new();
16245 let service = repositories.service();
16246 let domain = test_domain(149, "p1_10_rollback_study_domain");
16247 block_on(repositories.domains.save_domain(&domain))
16248 .expect("fixture domain should save");
16249 repositories.fail_next_registration_at(point);
16250
16251 let error = block_on(service.add_study(AddStudyRequest {
16252 name: "p1_10_rolled_back_study".to_string(),
16253 external_id: "rolled-back-study-ext".to_string(),
16254 study_type: "HDSS".to_string(),
16255 domain: DomainSelector::Name {
16256 name: domain.name.as_str().to_string(),
16257 },
16258 description: Some("Rollback fixture #failed".to_string()),
16259 }))
16260 .expect_err("injected atomic registration failure must fail Study registration");
16261
16262 assert!(matches!(error, AppError::Infrastructure(_)));
16263 assert!(
16264 block_on(repositories.studies.list_studies())
16265 .unwrap()
16266 .is_empty()
16267 );
16268 assert!(
16269 repositories
16270 .state()
16271 .study_domains
16272 .lock()
16273 .unwrap()
16274 .is_empty()
16275 );
16276 assert!(block_on(repositories.tags.list_tags()).unwrap().is_empty());
16277 assert!(
16278 repositories
16279 .state()
16280 .access_grants
16281 .lock()
16282 .unwrap()
16283 .is_empty()
16284 );
16285 assert!(repositories.state().custodians.lock().unwrap().is_empty());
16286 }
16287 }
16288
16289 #[test]
16290 fn failed_explicit_study_registration_uses_atomic_port() {
16291 let repositories = RegistrationRepositories::new();
16292 let service = repositories.service();
16293 let domain = test_domain(151, "explicit_rollback_study_domain");
16294 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16295 repositories.fail_next_registration_at(RegistrationFailurePoint::StudyCustodian);
16296 let mut study = test_study("rolled_back_explicit_study");
16297 study.description = Some("Rollback fixture #failed".to_string());
16298
16299 block_on(service.register_study(RegisterStudyRequest {
16300 study,
16301 domain_ids: vec![domain.domain_id],
16302 }))
16303 .expect_err("custodian failure must fail explicit Study registration");
16304
16305 assert!(
16306 block_on(repositories.studies.list_studies())
16307 .unwrap()
16308 .is_empty()
16309 );
16310 assert!(
16311 repositories
16312 .state()
16313 .study_domains
16314 .lock()
16315 .unwrap()
16316 .is_empty()
16317 );
16318 assert!(block_on(repositories.tags.list_tags()).unwrap().is_empty());
16319 assert!(
16320 repositories
16321 .state()
16322 .access_grants
16323 .lock()
16324 .unwrap()
16325 .is_empty()
16326 );
16327 assert!(repositories.state().custodians.lock().unwrap().is_empty());
16328 }
16329
16330 #[test]
16331 fn add_study_domain_facade_validates_missing_study_and_domain() {
16332 let repositories = RegistrationRepositories::new();
16333 let service = repositories.service();
16334 let domain = test_domain(47, "p1_10_link_domain");
16335 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16336 let study = block_on(service.add_study(AddStudyRequest {
16337 name: "p1_10_link_study".to_string(),
16338 external_id: "link-study-ext".to_string(),
16339 study_type: "1".to_string(),
16340 domain: DomainSelector::Name {
16341 name: "p1_10_link_domain".to_string(),
16342 },
16343 description: None,
16344 }))
16345 .expect("fixture study should save")
16346 .registration;
16347
16348 let missing_domain = block_on(service.add_study_domain(AddStudyDomainRequest {
16349 study: StudySelector::Name {
16350 domain: None,
16351 name: study.study.name.as_str().to_string(),
16352 },
16353 domain: DomainSelector::Name {
16354 name: "p1_10_missing_link_domain".to_string(),
16355 },
16356 }))
16357 .expect_err("missing domain should fail");
16358 let missing_study = block_on(service.add_study_domain(AddStudyDomainRequest {
16359 study: StudySelector::Name {
16360 domain: None,
16361 name: "p1_10_missing_link_study".to_string(),
16362 },
16363 domain: DomainSelector::Name {
16364 name: domain.name.as_str().to_string(),
16365 },
16366 }))
16367 .expect_err("missing study should fail");
16368
16369 assert!(
16370 matches!(missing_domain, AppError::NotFound(message) if message.contains("domain not found"))
16371 );
16372 assert!(
16373 matches!(missing_study, AppError::NotFound(message) if message.contains("study not found"))
16374 );
16375 }
16376
16377 #[test]
16378 fn register_domain_study_creates_link_and_reads_registration() {
16379 let repositories = RegistrationRepositories::new();
16380 let service = repositories.service();
16381 let domain = test_domain(42, "p1_10_domain");
16382 let study = test_study("p1_10_study");
16383
16384 let registration = block_on(service.register_domain_study(RegisterDomainStudyRequest {
16385 domain: domain.clone(),
16386 study: study.clone(),
16387 }))
16388 .expect("domain-study registration should succeed");
16389
16390 assert_eq!(registration.study, study);
16391 assert_eq!(registration.domains, vec![domain.clone()]);
16392 assert_creator_seeded_as_access_grant_and_primary_custodian(
16393 &repositories,
16394 registration.study.study_id,
16395 registration.study.created_by.as_deref().unwrap(),
16396 );
16397
16398 let by_name = block_on(service.get_study_registration(GetStudyRegistrationRequest {
16399 study_id: None,
16400 study_name: Some(study.name.clone()),
16401 }))
16402 .expect("registered study should be retrievable by name");
16403 assert_eq!(by_name, registration);
16404 }
16405
16406 #[test]
16407 fn register_domain_study_preserves_explicit_domain_and_study_ids() {
16408 let repositories = RegistrationRepositories::new();
16409 let service = repositories.service();
16410 let domain = test_domain(45, "p1_10_explicit_domain");
16411 let study = test_study("p1_10_explicit_study");
16412 let explicit_study_id = study.study_id;
16413
16414 let registration =
16415 block_on(service.register_domain_study(RegisterDomainStudyRequest { domain, study }))
16416 .expect("explicit domain-study registration should succeed");
16417
16418 assert_eq!(registration.domains[0].domain_id, DomainId(45));
16419 assert_eq!(registration.study.study_id, explicit_study_id);
16420 }
16421
16422 #[test]
16423 fn register_study_uses_atomic_port_for_creator_governance_defaults() {
16424 let repositories = RegistrationRepositories::new();
16425 let service = repositories.service();
16426 let domain = test_domain(43, "p1_10_unseeded_domain");
16427 block_on(repositories.domains.save_domain(&domain)).expect("fixture domain should save");
16428
16429 let registration = block_on(service.register_study(RegisterStudyRequest {
16430 study: test_study("p1_10_unseeded_study"),
16431 domain_ids: vec![domain.domain_id],
16432 }))
16433 .expect("study registration should create missing creator governance defaults");
16434
16435 assert_creator_seeded_as_access_grant_and_primary_custodian(
16436 &repositories,
16437 registration.study.study_id,
16438 registration.study.created_by.as_deref().unwrap(),
16439 );
16440 }
16441
16442 #[test]
16443 fn register_domain_study_reuses_existing_named_domain() {
16444 let repositories = RegistrationRepositories::new();
16445 let service = repositories.service();
16446 let existing_domain = test_domain(7, "p1_10_shared_domain");
16447 let requested_domain = test_domain(8, "p1_10_shared_domain");
16448 let study = test_study("p1_10_reuse_study");
16449
16450 block_on(repositories.domains.save_domain(&existing_domain))
16451 .expect("fixture domain should save");
16452
16453 let registration = block_on(service.register_domain_study(RegisterDomainStudyRequest {
16454 domain: requested_domain,
16455 study,
16456 }))
16457 .expect("domain-study registration should reuse the existing domain");
16458
16459 assert_eq!(registration.domains, vec![existing_domain]);
16460 }
16461
16462 #[test]
16463 fn get_study_registration_requires_one_selector() {
16464 let service = RegistrationRepositories::new().service();
16465
16466 let missing = block_on(service.get_study_registration(GetStudyRegistrationRequest {
16467 study_id: None,
16468 study_name: None,
16469 }))
16470 .expect_err("missing selector should fail");
16471 assert!(matches!(missing, AppError::Validation(_)));
16472
16473 let ambiguous = block_on(service.get_study_registration(GetStudyRegistrationRequest {
16474 study_id: Some(StudyId(uuid::Uuid::new_v4())),
16475 study_name: Some(nc_name("p1_10_ambiguous")),
16476 }))
16477 .expect_err("ambiguous selector should fail");
16478 assert!(matches!(ambiguous, AppError::Validation(_)));
16479 }
16480
16481 #[test]
16482 fn list_study_access_grants_returns_repository_visible_rows() {
16483 let repositories = RegistrationRepositories::new();
16484 let service = repositories.service();
16485 let study_id = StudyId(uuid::Uuid::new_v4());
16486 let hidden_study_id = StudyId(uuid::Uuid::new_v4());
16487 let visible = StudyAccessGrantRecord {
16488 study_id,
16489 user_id: "visible_user".to_string(),
16490 date_granted: None,
16491 granted_by: Some("primary_user".to_string()),
16492 };
16493 let hidden = StudyAccessGrantRecord {
16494 study_id: hidden_study_id,
16495 user_id: "hidden_user".to_string(),
16496 date_granted: None,
16497 granted_by: Some("primary_user".to_string()),
16498 };
16499 block_on(repositories.study_governance.save_access_grant(&visible))
16500 .expect("visible access fixture should save");
16501 block_on(repositories.study_governance.save_access_grant(&hidden))
16502 .expect("hidden access fixture should save");
16503
16504 let listed =
16505 block_on(service.list_study_access_grants(ListStudyAccessGrantsRequest { study_id }))
16506 .expect("study access grants should list through the app service");
16507
16508 assert_eq!(listed.study_id, study_id);
16509 assert_eq!(listed.grants, vec![visible]);
16510 }
16511
16512 #[test]
16513 fn grant_study_access_rejects_empty_target_before_repository_call() {
16514 let repositories = RegistrationRepositories::new();
16515 let service = repositories.service();
16516 let study_id = StudyId(uuid::Uuid::new_v4());
16517
16518 let error = block_on(service.grant_study_access(GrantStudyAccessRequest {
16519 study_id,
16520 target_user_id: " \t\n ".to_string(),
16521 }))
16522 .expect_err("empty study access target should fail validation");
16523
16524 assert!(matches!(error, AppError::Validation(_)));
16525 assert!(
16526 repositories
16527 .state()
16528 .grant_access_calls
16529 .lock()
16530 .expect("grant access calls lock should not be poisoned")
16531 .is_empty(),
16532 "validation should fail before the repository is called"
16533 );
16534 }
16535
16536 #[test]
16537 fn grant_study_access_trims_target_and_calls_named_repository_path() {
16538 let repositories = RegistrationRepositories::new();
16539 let service = repositories.service();
16540 let study_id = StudyId(uuid::Uuid::new_v4());
16541
16542 let granted = block_on(service.grant_study_access(GrantStudyAccessRequest {
16543 study_id,
16544 target_user_id: " orcid_0000-0000-0000-0001 ".to_string(),
16545 }))
16546 .expect("valid study access grant should call repository");
16547
16548 assert_eq!(granted.grant.study_id, study_id);
16549 assert_eq!(granted.grant.user_id, "orcid_0000-0000-0000-0001");
16550 assert_eq!(
16551 repositories
16552 .state()
16553 .grant_access_calls
16554 .lock()
16555 .expect("grant access calls lock should not be poisoned")
16556 .as_slice(),
16557 &[(study_id, "orcid_0000-0000-0000-0001".to_string())]
16558 );
16559 }
16560
16561 #[test]
16562 fn revoke_study_access_rejects_empty_target_before_repository_call() {
16563 let repositories = RegistrationRepositories::new();
16564 let service = repositories.service();
16565 let study_id = StudyId(uuid::Uuid::new_v4());
16566
16567 let error = block_on(service.revoke_study_access(RevokeStudyAccessRequest {
16568 study_id,
16569 target_user_id: " \t\n ".to_string(),
16570 }))
16571 .expect_err("empty study access target should fail validation");
16572
16573 assert!(matches!(error, AppError::Validation(_)));
16574 assert!(
16575 repositories
16576 .state()
16577 .revoke_access_calls
16578 .lock()
16579 .expect("revoke access calls lock should not be poisoned")
16580 .is_empty(),
16581 "validation should fail before the repository is called"
16582 );
16583 }
16584
16585 #[test]
16586 fn revoke_study_access_trims_target_and_calls_named_repository_path() {
16587 let repositories = RegistrationRepositories::new();
16588 let service = repositories.service();
16589 let study_id = StudyId(uuid::Uuid::new_v4());
16590 block_on(
16591 repositories
16592 .study_governance
16593 .save_access_grant(&StudyAccessGrantRecord {
16594 study_id,
16595 user_id: "orcid_0000-0000-0000-0002".to_string(),
16596 date_granted: None,
16597 granted_by: None,
16598 }),
16599 )
16600 .expect("study access fixture should save");
16601
16602 let revoked = block_on(service.revoke_study_access(RevokeStudyAccessRequest {
16603 study_id,
16604 target_user_id: " orcid_0000-0000-0000-0002 ".to_string(),
16605 }))
16606 .expect("valid study access revocation should call repository");
16607
16608 assert_eq!(revoked.study_id, study_id);
16609 assert_eq!(revoked.target_user_id, "orcid_0000-0000-0000-0002");
16610 assert_eq!(
16611 repositories
16612 .state()
16613 .revoke_access_calls
16614 .lock()
16615 .expect("revoke access calls lock should not be poisoned")
16616 .as_slice(),
16617 &[(study_id, "orcid_0000-0000-0000-0002".to_string())]
16618 );
16619 assert!(
16620 block_on(repositories.study_governance.list_access_grants(study_id))
16621 .expect("study access should list")
16622 .is_empty()
16623 );
16624 }
16625
16626 #[test]
16627 fn list_study_custodians_returns_repository_visible_rows() {
16628 let repositories = RegistrationRepositories::new();
16629 let service = repositories.service();
16630 let study_id = StudyId(uuid::Uuid::new_v4());
16631 let hidden_study_id = StudyId(uuid::Uuid::new_v4());
16632 let primary = StudyCustodianLinkRecord {
16633 study_id,
16634 user_id: "primary_user".to_string(),
16635 is_primary: true,
16636 date_granted: None,
16637 granted_by: Some("primary_user".to_string()),
16638 };
16639 let delegate = StudyCustodianLinkRecord {
16640 study_id,
16641 user_id: "delegate_user".to_string(),
16642 is_primary: false,
16643 date_granted: None,
16644 granted_by: Some("primary_user".to_string()),
16645 };
16646 let hidden = StudyCustodianLinkRecord {
16647 study_id: hidden_study_id,
16648 user_id: "hidden_user".to_string(),
16649 is_primary: true,
16650 date_granted: None,
16651 granted_by: Some("hidden_user".to_string()),
16652 };
16653 block_on(repositories.study_governance.save_custodian(&primary))
16654 .expect("primary custodian fixture should save");
16655 block_on(repositories.study_governance.save_custodian(&delegate))
16656 .expect("delegate custodian fixture should save");
16657 block_on(repositories.study_governance.save_custodian(&hidden))
16658 .expect("hidden custodian fixture should save");
16659
16660 let listed =
16661 block_on(service.list_study_custodians(ListStudyCustodiansRequest { study_id }))
16662 .expect("study custodians should list through the app service");
16663
16664 assert_eq!(listed.study_id, study_id);
16665 assert_eq!(listed.custodians, vec![primary, delegate]);
16666 }
16667
16668 #[test]
16669 fn add_delegate_custodian_rejects_empty_target_before_repository_call() {
16670 let repositories = RegistrationRepositories::new();
16671 let service = repositories.service();
16672 let study_id = StudyId(uuid::Uuid::new_v4());
16673
16674 let error = block_on(service.add_delegate_custodian(AddDelegateCustodianRequest {
16675 study_id,
16676 target_user_id: " \t\n ".to_string(),
16677 }))
16678 .expect_err("empty delegate custodian target should fail validation");
16679
16680 assert!(matches!(error, AppError::Validation(_)));
16681 assert!(
16682 repositories
16683 .state()
16684 .add_delegate_custodian_calls
16685 .lock()
16686 .expect("delegate custodian calls lock should not be poisoned")
16687 .is_empty(),
16688 "validation should fail before the repository is called"
16689 );
16690 }
16691
16692 #[test]
16693 fn add_delegate_custodian_trims_target_and_calls_named_repository_path() {
16694 let repositories = RegistrationRepositories::new();
16695 let service = repositories.service();
16696 let study_id = StudyId(uuid::Uuid::new_v4());
16697
16698 let added = block_on(service.add_delegate_custodian(AddDelegateCustodianRequest {
16699 study_id,
16700 target_user_id: " orcid_0000-0000-0000-0002 ".to_string(),
16701 }))
16702 .expect("valid delegate custodian should call repository");
16703
16704 assert_eq!(added.custodian.study_id, study_id);
16705 assert_eq!(added.custodian.user_id, "orcid_0000-0000-0000-0002");
16706 assert!(!added.custodian.is_primary);
16707 assert_eq!(
16708 repositories
16709 .state()
16710 .add_delegate_custodian_calls
16711 .lock()
16712 .expect("delegate custodian calls lock should not be poisoned")
16713 .as_slice(),
16714 &[(study_id, "orcid_0000-0000-0000-0002".to_string())]
16715 );
16716 }
16717
16718 #[test]
16719 fn transfer_primary_custodianship_rejects_empty_target_before_repository_call() {
16720 let repositories = RegistrationRepositories::new();
16721 let service = repositories.service();
16722 let study_id = StudyId(uuid::Uuid::new_v4());
16723
16724 let error = block_on(service.transfer_primary_custodianship(
16725 TransferPrimaryCustodianshipRequest {
16726 study_id,
16727 target_user_id: " \t\n ".to_string(),
16728 },
16729 ))
16730 .expect_err("empty primary transfer target should fail validation");
16731
16732 assert!(matches!(error, AppError::Validation(_)));
16733 assert!(
16734 repositories
16735 .state()
16736 .transfer_primary_custodianship_calls
16737 .lock()
16738 .expect("primary transfer calls lock should not be poisoned")
16739 .is_empty(),
16740 "validation should fail before the repository is called"
16741 );
16742 }
16743
16744 #[test]
16745 fn transfer_primary_custodianship_trims_target_and_calls_named_repository_path() {
16746 let repositories = RegistrationRepositories::new();
16747 let service = repositories.service();
16748 let study_id = StudyId(uuid::Uuid::new_v4());
16749
16750 let transferred = block_on(service.transfer_primary_custodianship(
16751 TransferPrimaryCustodianshipRequest {
16752 study_id,
16753 target_user_id: " orcid_0000-0000-0000-0003 ".to_string(),
16754 },
16755 ))
16756 .expect("valid primary transfer should call repository");
16757
16758 assert_eq!(transferred.primary_custodian.study_id, study_id);
16759 assert_eq!(
16760 transferred.primary_custodian.user_id,
16761 "orcid_0000-0000-0000-0003"
16762 );
16763 assert!(transferred.primary_custodian.is_primary);
16764 assert_eq!(
16765 repositories
16766 .state()
16767 .transfer_primary_custodianship_calls
16768 .lock()
16769 .expect("primary transfer calls lock should not be poisoned")
16770 .as_slice(),
16771 &[(study_id, "orcid_0000-0000-0000-0003".to_string())]
16772 );
16773 }
16774
16775 #[test]
16776 fn remove_delegate_custodian_rejects_empty_target_before_repository_call() {
16777 let repositories = RegistrationRepositories::new();
16778 let service = repositories.service();
16779 let study_id = StudyId(uuid::Uuid::new_v4());
16780
16781 let error = block_on(
16782 service.remove_delegate_custodian(RemoveDelegateCustodianRequest {
16783 study_id,
16784 target_user_id: " \t\n ".to_string(),
16785 }),
16786 )
16787 .expect_err("empty delegate custodian target should fail validation");
16788
16789 assert!(matches!(error, AppError::Validation(_)));
16790 assert!(
16791 repositories
16792 .state()
16793 .remove_delegate_custodian_calls
16794 .lock()
16795 .expect("delegate custodian removal calls lock should not be poisoned")
16796 .is_empty(),
16797 "validation should fail before the repository is called"
16798 );
16799 }
16800
16801 #[test]
16802 fn remove_delegate_custodian_trims_target_and_calls_named_repository_path() {
16803 let repositories = RegistrationRepositories::new();
16804 let service = repositories.service();
16805 let study_id = StudyId(uuid::Uuid::new_v4());
16806 block_on(
16807 repositories
16808 .study_governance
16809 .save_custodian(&StudyCustodianLinkRecord {
16810 study_id,
16811 user_id: "orcid_0000-0000-0000-0004".to_string(),
16812 is_primary: false,
16813 date_granted: None,
16814 granted_by: None,
16815 }),
16816 )
16817 .expect("delegate custodian fixture should save");
16818
16819 let removed = block_on(
16820 service.remove_delegate_custodian(RemoveDelegateCustodianRequest {
16821 study_id,
16822 target_user_id: " orcid_0000-0000-0000-0004 ".to_string(),
16823 }),
16824 )
16825 .expect("valid delegate removal should call repository");
16826
16827 assert_eq!(removed.study_id, study_id);
16828 assert_eq!(removed.target_user_id, "orcid_0000-0000-0000-0004");
16829 assert_eq!(
16830 repositories
16831 .state()
16832 .remove_delegate_custodian_calls
16833 .lock()
16834 .expect("delegate custodian removal calls lock should not be poisoned")
16835 .as_slice(),
16836 &[(study_id, "orcid_0000-0000-0000-0004".to_string())]
16837 );
16838 assert!(
16839 block_on(repositories.study_governance.list_custodians(study_id))
16840 .expect("study custodians should list")
16841 .is_empty()
16842 );
16843 }
16844
16845 #[test]
16846 fn entity_definition_registration_uses_repository_assigned_ids_and_uuids() {
16847 let repositories = EntityGraphRepositories::new();
16848 let service = repositories.service();
16849 let domain = test_domain(91, "p1_14_entity_domain");
16850 repositories.seed_domain(domain.clone());
16851
16852 let person = block_on(service.register_entity(RegisterEntityRequest {
16853 entity: NewEntityRecord {
16854 domain_id: domain.domain_id,
16855 name: nc_name("person"),
16856 description: Some("Person entity #entity_tag".to_string()),
16857 agent_instructions: Some("Person instructions".to_string()),
16858 ontology_namespace: Some("https://example.test/ontology".to_string()),
16859 ontology_class: Some("Person".to_string()),
16860 },
16861 }))
16862 .expect("new entity should be registered with an assigned id");
16863 let household = block_on(service.register_entity(RegisterEntityRequest {
16864 entity: NewEntityRecord {
16865 domain_id: domain.domain_id,
16866 name: nc_name("household"),
16867 description: Some("Household entity".to_string()),
16868 agent_instructions: None,
16869 ontology_namespace: None,
16870 ontology_class: None,
16871 },
16872 }))
16873 .expect("second entity should be registered with an assigned id");
16874
16875 assert_eq!(person.entity_id, EntityId(1));
16876 assert_eq!(household.entity_id, EntityId(2));
16877 assert!(person.uuid.is_some());
16878 assert!(household.uuid.is_some());
16879 assert_ne!(person.uuid, household.uuid);
16880 assert_eq!(person.description.as_deref(), Some("Person entity"));
16881
16882 let member_of = block_on(
16883 service.register_entity_relation(RegisterEntityRelationRequest {
16884 relation: NewEntityRelationRecord {
16885 subject_entity_id: person.entity_id,
16886 object_entity_id: household.entity_id,
16887 domain_id: domain.domain_id,
16888 name: nc_name("member_of"),
16889 description: Some("Membership relation #relation_tag".to_string()),
16890 agent_instructions: None,
16891 ontology_namespace: None,
16892 ontology_class: None,
16893 },
16894 }),
16895 )
16896 .expect("new entity relation should be registered with an assigned id");
16897
16898 assert_eq!(member_of.entity_relation_id, EntityRelationId(1));
16899 assert_eq!(member_of.subject_entity_id, person.entity_id);
16900 assert_eq!(member_of.object_entity_id, household.entity_id);
16901 assert!(member_of.uuid.is_some());
16902 assert_eq!(
16903 member_of.description.as_deref(),
16904 Some("Membership relation")
16905 );
16906
16907 let explicit_uuid = uuid::Uuid::parse_str("018f0c7a-0000-7000-8000-000000000077")
16908 .expect("fixture uuid should parse");
16909 let explicit = block_on(repositories.entities.save_entity(&EntityRecord {
16910 entity_id: EntityId(77),
16911 domain_id: domain.domain_id,
16912 uuid: Some(explicit_uuid),
16913 name: nc_name("fixture_entity"),
16914 description: Some("explicit fixture entity".to_string()),
16915 agent_instructions: None,
16916 ontology_namespace: None,
16917 ontology_class: None,
16918 }))
16919 .expect("explicit entity save should preserve fixture identity");
16920 assert_eq!(explicit.entity_id, EntityId(77));
16921 assert_eq!(explicit.uuid, Some(explicit_uuid));
16922 }
16923
16924 #[test]
16925 fn model_definition_facades_accept_id_selectors_for_entities_relations_and_domains() {
16926 let repositories = EntityGraphRepositories::new();
16927 let service = repositories.service();
16928 let domain = test_domain(98, "p1_20i_model_id_domain");
16929 repositories.seed_domain(domain.clone());
16930
16931 let person = block_on(service.add_entity(AddEntityRequest {
16932 domain: DomainSelector::Id {
16933 domain_id: domain.domain_id,
16934 },
16935 name: "person".to_string(),
16936 description: None,
16937 agent_instructions: None,
16938 ontology_namespace: None,
16939 ontology_class: None,
16940 }))
16941 .expect("entity add should resolve domain id");
16942 let household = block_on(service.add_entity(AddEntityRequest {
16943 domain: DomainSelector::Id {
16944 domain_id: domain.domain_id,
16945 },
16946 name: "household".to_string(),
16947 description: None,
16948 agent_instructions: None,
16949 ontology_namespace: None,
16950 ontology_class: None,
16951 }))
16952 .expect("second entity add should resolve domain id");
16953 let member_of = block_on(service.add_entity_relation(AddEntityRelationRequest {
16954 domain: DomainSelector::Id {
16955 domain_id: domain.domain_id,
16956 },
16957 name: "member_of".to_string(),
16958 subject: person.name.as_str().to_string(),
16959 object: household.name.as_str().to_string(),
16960 description: None,
16961 agent_instructions: None,
16962 ontology_namespace: None,
16963 ontology_class: None,
16964 }))
16965 .expect("relation add should resolve domain id");
16966
16967 let entity_by_id = block_on(service.get_entity(GetEntityRequest {
16968 entity: EntitySelector::Id {
16969 entity_id: person.entity_id,
16970 },
16971 }))
16972 .expect("entity id selector should read")
16973 .expect("entity should exist");
16974 let entity_by_nested_domain_id = block_on(service.get_entity(GetEntityRequest {
16975 entity: EntitySelector::Name {
16976 domain: DomainSelector::Id {
16977 domain_id: domain.domain_id,
16978 },
16979 name: "person".to_string(),
16980 },
16981 }))
16982 .expect("nested domain id selector should read")
16983 .expect("entity should exist by nested domain id");
16984 let relation_by_id = block_on(service.get_entity_relation(GetEntityRelationRequest {
16985 relation: EntityRelationSelector::Id {
16986 entity_relation_id: member_of.entity_relation_id,
16987 },
16988 }))
16989 .expect("relation id selector should read")
16990 .expect("relation should exist");
16991 let relation_by_nested_domain_id =
16992 block_on(service.get_entity_relation(GetEntityRelationRequest {
16993 relation: EntityRelationSelector::Name {
16994 domain: DomainSelector::Id {
16995 domain_id: domain.domain_id,
16996 },
16997 name: "member_of".to_string(),
16998 },
16999 }))
17000 .expect("nested domain id selector should read relation")
17001 .expect("relation should exist by nested domain id");
17002 let missing_entity = block_on(service.get_entity(GetEntityRequest {
17003 entity: EntitySelector::Id {
17004 entity_id: EntityId(404),
17005 },
17006 }))
17007 .expect("missing entity id should not fail infrastructure");
17008
17009 assert_eq!(entity_by_id, person);
17010 assert_eq!(entity_by_nested_domain_id, person);
17011 assert_eq!(relation_by_id, member_of);
17012 assert_eq!(relation_by_nested_domain_id, member_of);
17013 assert_eq!(missing_entity, None);
17014 }
17015
17016 #[test]
17017 fn semantic_delete_model_definition_id_selectors_match_name_selectors_and_execute() {
17018 let repositories = EntityGraphRepositories::new();
17019 let service = repositories.service();
17020 let domain = test_domain(99, "p1_20i_semantic_model_delete_domain");
17021 repositories.seed_domain(domain.clone());
17022
17023 let lone_entity = block_on(service.add_entity(AddEntityRequest {
17024 domain: DomainSelector::Id {
17025 domain_id: domain.domain_id,
17026 },
17027 name: "delete_me_entity".to_string(),
17028 description: None,
17029 agent_instructions: None,
17030 ontology_namespace: None,
17031 ontology_class: None,
17032 }))
17033 .expect("entity add should succeed");
17034 let entity_by_name = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
17035 target: SemanticDeleteTarget::Entity {
17036 entity: EntitySelector::Name {
17037 domain: DomainSelector::Name {
17038 name: domain.name.as_str().to_string(),
17039 },
17040 name: lone_entity.name.as_str().to_string(),
17041 },
17042 },
17043 reason: "remove test entity".to_string(),
17044 actor: Some("curator@example.org".to_string()),
17045 dry_run: true,
17046 }))
17047 .expect("entity name delete plan should succeed")
17048 .plan
17049 .expect("entity name plan should be accepted");
17050 let entity_by_id = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
17051 target: SemanticDeleteTarget::Entity {
17052 entity: EntitySelector::Id {
17053 entity_id: lone_entity.entity_id,
17054 },
17055 },
17056 reason: "remove test entity".to_string(),
17057 actor: Some("curator@example.org".to_string()),
17058 dry_run: true,
17059 }))
17060 .expect("entity id delete plan should succeed")
17061 .plan
17062 .expect("entity id plan should be accepted");
17063 assert_eq!(entity_by_id.target_identity, entity_by_name.target_identity);
17064 assert_eq!(
17065 entity_by_id.dependency_summary,
17066 entity_by_name.dependency_summary
17067 );
17068 let entity_deleted = block_on(service.execute_semantic_delete(
17069 ExecuteSemanticDeleteRequest {
17070 plan: entity_by_id,
17071 confirmed: true,
17072 },
17073 ))
17074 .expect("entity id delete should execute");
17075 assert!(entity_deleted.deleted);
17076
17077 let subject = block_on(service.add_entity(AddEntityRequest {
17078 domain: DomainSelector::Id {
17079 domain_id: domain.domain_id,
17080 },
17081 name: "subject_entity".to_string(),
17082 description: None,
17083 agent_instructions: None,
17084 ontology_namespace: None,
17085 ontology_class: None,
17086 }))
17087 .expect("subject entity add should succeed");
17088 let object = block_on(service.add_entity(AddEntityRequest {
17089 domain: DomainSelector::Id {
17090 domain_id: domain.domain_id,
17091 },
17092 name: "object_entity".to_string(),
17093 description: None,
17094 agent_instructions: None,
17095 ontology_namespace: None,
17096 ontology_class: None,
17097 }))
17098 .expect("object entity add should succeed");
17099 let relation = block_on(service.add_entity_relation(AddEntityRelationRequest {
17100 domain: DomainSelector::Id {
17101 domain_id: domain.domain_id,
17102 },
17103 name: "delete_me_relation".to_string(),
17104 subject: subject.name.as_str().to_string(),
17105 object: object.name.as_str().to_string(),
17106 description: None,
17107 agent_instructions: None,
17108 ontology_namespace: None,
17109 ontology_class: None,
17110 }))
17111 .expect("relation add should succeed");
17112 let relation_by_name = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
17113 target: SemanticDeleteTarget::EntityRelation {
17114 relation: EntityRelationSelector::Name {
17115 domain: DomainSelector::Name {
17116 name: domain.name.as_str().to_string(),
17117 },
17118 name: relation.name.as_str().to_string(),
17119 },
17120 },
17121 reason: "remove test relation".to_string(),
17122 actor: Some("curator@example.org".to_string()),
17123 dry_run: true,
17124 }))
17125 .expect("relation name delete plan should succeed")
17126 .plan
17127 .expect("relation name plan should be accepted");
17128 let relation_by_id = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
17129 target: SemanticDeleteTarget::EntityRelation {
17130 relation: EntityRelationSelector::Id {
17131 entity_relation_id: relation.entity_relation_id,
17132 },
17133 },
17134 reason: "remove test relation".to_string(),
17135 actor: Some("curator@example.org".to_string()),
17136 dry_run: true,
17137 }))
17138 .expect("relation id delete plan should succeed")
17139 .plan
17140 .expect("relation id plan should be accepted");
17141 assert_eq!(
17142 relation_by_id.target_identity,
17143 relation_by_name.target_identity
17144 );
17145 assert_eq!(
17146 relation_by_id.dependency_summary,
17147 relation_by_name.dependency_summary
17148 );
17149 let relation_deleted = block_on(service.execute_semantic_delete(
17150 ExecuteSemanticDeleteRequest {
17151 plan: relation_by_id,
17152 confirmed: true,
17153 },
17154 ))
17155 .expect("relation id delete should execute");
17156 assert!(relation_deleted.deleted);
17157 }
17158
17159 #[test]
17160 fn study_entity_graph_workflows_use_returned_instance_ids_for_mappings_and_links() {
17161 let repositories = EntityGraphRepositories::new();
17162 let service = repositories.service();
17163 let study = test_study("p1_14_entity_graph_study");
17164 let domain = test_domain(92, "p1_14_entity_graph_domain");
17165 repositories.seed_study(study.clone());
17166 repositories.seed_domain(domain.clone());
17167
17168 let person = block_on(repositories.entities.create_entity(&NewEntityRecord {
17169 domain_id: domain.domain_id,
17170 name: nc_name("graph_person"),
17171 description: None,
17172 agent_instructions: None,
17173 ontology_namespace: None,
17174 ontology_class: None,
17175 }))
17176 .expect("person entity should persist");
17177 let household = block_on(repositories.entities.create_entity(&NewEntityRecord {
17178 domain_id: domain.domain_id,
17179 name: nc_name("graph_household"),
17180 description: None,
17181 agent_instructions: None,
17182 ontology_namespace: None,
17183 ontology_class: None,
17184 }))
17185 .expect("household entity should persist");
17186 let member_of = block_on(repositories.entities.create_entity_relation(
17187 &NewEntityRelationRecord {
17188 subject_entity_id: person.entity_id,
17189 object_entity_id: household.entity_id,
17190 domain_id: domain.domain_id,
17191 name: nc_name("graph_member_of"),
17192 description: None,
17193 agent_instructions: None,
17194 ontology_namespace: None,
17195 ontology_class: None,
17196 },
17197 ))
17198 .expect("entity relation should persist");
17199
17200 let subject = block_on(service.ensure_study_entity_instance(
17201 EnsureStudyEntityInstanceRequest {
17202 study_id: study.study_id,
17203 entity_id: person.entity_id,
17204 external_id: "person-1".to_string(),
17205 canonical_uuid: None,
17206 label: Some("Person 1".to_string()),
17207 note: None,
17208 transformation_id: TransformationId(10),
17209 },
17210 ))
17211 .expect("subject entity instance should be created and mapped");
17212 let object = block_on(service.ensure_study_entity_instance(
17213 EnsureStudyEntityInstanceRequest {
17214 study_id: study.study_id,
17215 entity_id: household.entity_id,
17216 external_id: "household-1".to_string(),
17217 canonical_uuid: None,
17218 label: Some("Household 1".to_string()),
17219 note: None,
17220 transformation_id: TransformationId(10),
17221 },
17222 ))
17223 .expect("object entity instance should be created and mapped");
17224
17225 assert_eq!(subject.instance.instance_id, 1);
17226 assert_eq!(
17227 subject.mapping.entity_instance_id,
17228 subject.instance.instance_id
17229 );
17230 assert_eq!(object.instance.instance_id, 2);
17231 assert_eq!(
17232 object.mapping.entity_instance_id,
17233 object.instance.instance_id
17234 );
17235 assert_ne!(subject.instance.uuid, uuid::Uuid::nil());
17236
17237 let relation = block_on(service.ensure_study_relation_instance(
17238 EnsureStudyRelationInstanceRequest {
17239 study_id: study.study_id,
17240 entity_relation_id: member_of.entity_relation_id,
17241 external_id: "membership-1".to_string(),
17242 subject_entity_instance_id: subject.instance.instance_id,
17243 object_entity_instance_id: object.instance.instance_id,
17244 valid_from: None,
17245 valid_to: None,
17246 note: Some("linked by assigned entity instance ids".to_string()),
17247 transformation_id: TransformationId(10),
17248 },
17249 ))
17250 .expect("relation instance should be created and mapped");
17251
17252 assert_eq!(relation.instance.relation_instance_id, 1);
17253 assert_eq!(
17254 relation.mapping.relation_instance_id,
17255 relation.instance.relation_instance_id
17256 );
17257 assert_ne!(relation.instance.uuid, uuid::Uuid::nil());
17258
17259 block_on(
17260 service.link_dataset_version_entities(LinkDatasetVersionEntitiesRequest {
17261 links: vec![DatasetVersionEntityRecord {
17262 version_id: VersionId(uuid::Uuid::from_u128(101)),
17263 entity_instance_id: subject.instance.instance_id,
17264 entity_variable_id: VariableId(501),
17265 transformation_id: TransformationId(10),
17266 }],
17267 }),
17268 )
17269 .expect("dataset entity link should use the returned entity instance id");
17270 block_on(service.link_dataset_version_relation_instances(
17271 LinkDatasetVersionRelationInstancesRequest {
17272 links: vec![DatasetVersionRelationInstanceRecord {
17273 version_id: VersionId(uuid::Uuid::from_u128(102)),
17274 relation_instance_id: relation.instance.relation_instance_id,
17275 subject_variable_id: VariableId(501),
17276 object_variable_id: VariableId(502),
17277 relation_variable_id: Some(VariableId(503)),
17278 transformation_id: TransformationId(10),
17279 }],
17280 },
17281 ))
17282 .expect("dataset relation link should use the returned relation instance id");
17283
17284 let entity_links = repositories
17285 .state
17286 .dataset_entity_links
17287 .lock()
17288 .expect("entity link lock should not be poisoned");
17289 assert_eq!(
17290 entity_links[0].entity_instance_id,
17291 subject.instance.instance_id
17292 );
17293 let relation_links = repositories
17294 .state
17295 .dataset_relation_links
17296 .lock()
17297 .expect("relation link lock should not be poisoned");
17298 assert_eq!(
17299 relation_links[0].relation_instance_id,
17300 relation.instance.relation_instance_id
17301 );
17302
17303 let explicit_uuid = uuid::Uuid::parse_str("018f0c7a-0000-7000-8000-000000000088")
17304 .expect("fixture uuid should parse");
17305 let explicit_instance = block_on(repositories.entities.save_entity_instance(
17306 &EntityInstanceWriteRecord {
17307 instance_id: Some(88),
17308 uuid: Some(explicit_uuid),
17309 entity_id: person.entity_id,
17310 label: Some("Explicit instance".to_string()),
17311 note: None,
17312 transformation_id: TransformationId(11),
17313 },
17314 ))
17315 .expect("explicit entity instance save should preserve fixture identity");
17316 assert_eq!(explicit_instance.instance_id, 88);
17317 assert_eq!(explicit_instance.uuid, explicit_uuid);
17318
17319 let explicit_relation_uuid = uuid::Uuid::parse_str("018f0c7a-0000-7000-8000-000000000099")
17320 .expect("fixture uuid should parse");
17321 let explicit_relation = block_on(repositories.entities.save_relation_instance(
17322 &RelationInstanceWriteRecord {
17323 relation_instance_id: Some(99),
17324 uuid: Some(explicit_relation_uuid),
17325 entity_relation_id: member_of.entity_relation_id,
17326 entity_instance_id_1: subject.instance.instance_id,
17327 entity_instance_id_2: object.instance.instance_id,
17328 valid_from: None,
17329 valid_to: None,
17330 note: Some("Explicit relation instance".to_string()),
17331 transformation_id: TransformationId(11),
17332 },
17333 ))
17334 .expect("explicit relation instance save should preserve fixture identity");
17335 assert_eq!(explicit_relation.relation_instance_id, 99);
17336 assert_eq!(explicit_relation.uuid, explicit_relation_uuid);
17337 }
17338
17339 #[test]
17340 fn semantic_entity_commands_map_instances_and_link_dataset_variables() {
17341 let entity_repositories = EntityGraphRepositories::new();
17342 let asset_repositories = AssetWorkflowRepositories::new();
17343 let study = test_study("cli_entity_command_study");
17344 let domain = test_domain(194, "cli_entity_command_domain");
17345 entity_repositories.seed_study(study.clone());
17346 entity_repositories.seed_domain(domain.clone());
17347 asset_repositories.seed_study(study.clone());
17348 asset_repositories.seed_domain(domain.clone());
17349
17350 let service = AppService::new(AppRepositories {
17351 registrations: Arc::new(NoopCatalogueRegistrationRepository),
17352 domains: entity_repositories.domains.clone(),
17353 studies: entity_repositories.studies.clone(),
17354 study_domains: Arc::new(NoopStudyDomainRepository),
17355 study_governance: Arc::new(NoopStudyGovernanceRepository),
17356 assets: asset_repositories.assets.clone(),
17357 variables: asset_repositories.variables.clone(),
17358 vocabularies: asset_repositories.vocabularies.clone(),
17359 entities: entity_repositories.entities.clone(),
17360 transformations: Arc::new(NoopTransformationRepository),
17361 tags: Arc::new(NoopTagRepository),
17362 });
17363
17364 let entity = block_on(service.add_entity(AddEntityRequest {
17365 domain: domain.name.as_str().into(),
17366 name: "participant".to_string(),
17367 description: Some("Participant".to_string()),
17368 agent_instructions: None,
17369 ontology_namespace: None,
17370 ontology_class: None,
17371 }))
17372 .expect("entity command should create a definition");
17373 let instance = block_on(service.add_entity_instance(AddEntityInstanceRequest {
17374 domain: domain.name.as_str().to_string(),
17375 entity: entity.name.as_str().to_string(),
17376 transformation_id: TransformationId(42),
17377 label: Some("Participant 20010001".to_string()),
17378 note: None,
17379 }))
17380 .expect("entity instance command should create canonical instance");
17381 let mapping = block_on(service.add_study_entity_instance_mapping(
17382 AddStudyEntityInstanceMappingRequest {
17383 study: study.name.as_str().to_string(),
17384 instance_id: instance.instance_id,
17385 external_id: "20010001".to_string(),
17386 transformation_id: TransformationId(42),
17387 },
17388 ))
17389 .expect("mapping command should preserve study scope");
17390 assert_eq!(mapping.mapping.study_id, study.study_id);
17391 assert_eq!(mapping.mapping.entity_instance_id, instance.instance_id);
17392 let mapping_list = block_on(service.list_study_entity_instance_mappings(
17393 ListStudyEntityInstanceMappingsRequest {
17394 study: study.name.as_str().to_string(),
17395 domain: Some(domain.name.as_str().to_string()),
17396 entity: Some(entity.name.as_str().to_string()),
17397 },
17398 ))
17399 .expect("mapping list readback should succeed");
17400 assert_eq!(mapping_list.study, study);
17401 assert_eq!(mapping_list.mappings.len(), 1);
17402 assert_eq!(mapping_list.mappings[0].domain, domain);
17403 assert_eq!(mapping_list.mappings[0].entity, entity);
17404 assert_eq!(mapping_list.mappings[0].instance, instance);
17405 assert_eq!(mapping_list.mappings[0].mapping, mapping.mapping);
17406
17407 let asset = test_asset(study.study_id, "participants");
17408 let version = block_on(asset_repositories.assets.save_asset(&asset))
17409 .and_then(|asset| {
17410 let version = test_asset_version(asset.asset_id, 1, 0, 0, "participants v1");
17411 block_on(asset_repositories.assets.save_asset_version(&version))
17412 })
17413 .expect("dataset asset and version should save");
17414 asset_repositories.seed_dataset(DatasetRecord {
17415 dataset_id: version.version_id,
17416 });
17417 let participant_id = test_variable(
17418 501,
17419 domain.domain_id.0,
17420 "participant_id",
17421 KeyRole::External,
17422 None,
17423 );
17424 asset_repositories.seed_variable(participant_id.clone());
17425 block_on(asset_repositories.variables.link_dataset_variable(
17426 version.version_id,
17427 participant_id.variable_id,
17428 KeyRole::External,
17429 ))
17430 .expect("dataset variable fixture should link");
17431
17432 let asset_link = block_on(service.add_entity_asset_link(AddEntityAssetLinkRequest {
17433 study: study.name.as_str().to_string(),
17434 instance_id: instance.instance_id,
17435 asset: asset.name.as_str().to_string(),
17436 version: Some("latest".to_string()),
17437 transformation_id: TransformationId(44),
17438 }))
17439 .expect("asset link command should resolve latest version");
17440 let links_by_instance = block_on(service.list_entity_asset_links(
17441 ListEntityAssetLinksRequest {
17442 instance_id: Some(instance.instance_id),
17443 study: None,
17444 asset: None,
17445 version: None,
17446 },
17447 ))
17448 .expect("asset links should list by instance");
17449 assert_eq!(links_by_instance.links.len(), 1);
17450 assert_eq!(links_by_instance.links[0].link, asset_link);
17451 assert_eq!(links_by_instance.links[0].asset, asset);
17452 assert_eq!(links_by_instance.links[0].version, version);
17453 assert_eq!(
17454 links_by_instance.links[0].study_mapping,
17455 Some(mapping.mapping.clone())
17456 );
17457 let links_by_asset = block_on(service.list_entity_asset_links(
17458 ListEntityAssetLinksRequest {
17459 instance_id: None,
17460 study: Some(study.name.as_str().to_string()),
17461 asset: Some(asset.name.as_str().to_string()),
17462 version: Some("latest".to_string()),
17463 },
17464 ))
17465 .expect("asset links should list by version");
17466 assert_eq!(links_by_asset.resolved_asset, Some(asset.clone()));
17467 assert_eq!(links_by_asset.resolved_version, Some(version.clone()));
17468 assert_eq!(links_by_asset.links.len(), 1);
17469 assert_eq!(links_by_asset.links[0].link, asset_link);
17470
17471 let link = block_on(
17472 service.add_entity_dataset_link(AddEntityDatasetLinkRequest {
17473 study: study.name.as_str().to_string(),
17474 instance_id: instance.instance_id,
17475 dataset: asset.name.as_str().to_string(),
17476 variable: participant_id.name.as_str().to_string(),
17477 version: Some("latest".to_string()),
17478 transformation_id: TransformationId(43),
17479 }),
17480 )
17481 .expect("dataset link command should resolve the dataset variable");
17482 assert_eq!(link.version_id, version.version_id);
17483 assert_eq!(link.entity_instance_id, instance.instance_id);
17484 assert_eq!(link.entity_variable_id, participant_id.variable_id);
17485 let datasets_by_instance = block_on(service.list_entity_instance_datasets(
17486 ListEntityInstanceDatasetsRequest {
17487 instance_id: instance.instance_id,
17488 },
17489 ))
17490 .expect("entity instance datasets should list");
17491 assert_eq!(datasets_by_instance.links.len(), 1);
17492 assert_eq!(datasets_by_instance.links[0].dataset, asset);
17493 assert_eq!(datasets_by_instance.links[0].version, version);
17494 assert_eq!(datasets_by_instance.links[0].variable, participant_id);
17495 assert_eq!(datasets_by_instance.links[0].link, link);
17496 let precise_dataset_link = block_on(service.get_entity_dataset_link(
17497 GetEntityDatasetLinkRequest {
17498 instance_id: instance.instance_id,
17499 study: Some(study.name.as_str().to_string()),
17500 dataset: asset.name.as_str().to_string(),
17501 version: Some("latest".to_string()),
17502 },
17503 ))
17504 .expect("precise entity dataset link readback should succeed")
17505 .expect("precise entity dataset link should exist");
17506 assert_eq!(precise_dataset_link.link, link);
17507 let dataset_links = block_on(service.list_entity_dataset_links(
17508 ListEntityDatasetLinksRequest {
17509 study: Some(study.name.as_str().to_string()),
17510 dataset: asset.name.as_str().to_string(),
17511 version: Some("latest".to_string()),
17512 },
17513 ))
17514 .expect("entity dataset links should list by dataset version");
17515 assert_eq!(dataset_links.resolved_dataset, Some(asset));
17516 assert_eq!(dataset_links.resolved_version, Some(version));
17517 assert_eq!(dataset_links.links.len(), 1);
17518 assert_eq!(dataset_links.links[0].link, link);
17519 }
17520
17521 #[test]
17522 fn semantic_relation_readback_lists_mappings_asset_links_and_dataset_links() {
17523 let entity_repositories = EntityGraphRepositories::new();
17524 let asset_repositories = AssetWorkflowRepositories::new();
17525 let study = test_study("relation_readback_study");
17526 let domain = test_domain(195, "relation_readback_domain");
17527 entity_repositories.seed_study(study.clone());
17528 entity_repositories.seed_domain(domain.clone());
17529 asset_repositories.seed_study(study.clone());
17530 asset_repositories.seed_domain(domain.clone());
17531
17532 let service = AppService::new(AppRepositories {
17533 registrations: Arc::new(NoopCatalogueRegistrationRepository),
17534 domains: entity_repositories.domains.clone(),
17535 studies: entity_repositories.studies.clone(),
17536 study_domains: Arc::new(NoopStudyDomainRepository),
17537 study_governance: Arc::new(NoopStudyGovernanceRepository),
17538 assets: asset_repositories.assets.clone(),
17539 variables: asset_repositories.variables.clone(),
17540 vocabularies: asset_repositories.vocabularies.clone(),
17541 entities: entity_repositories.entities.clone(),
17542 transformations: Arc::new(NoopTransformationRepository),
17543 tags: Arc::new(NoopTagRepository),
17544 });
17545
17546 let person = block_on(service.add_entity(AddEntityRequest {
17547 domain: domain.name.as_str().into(),
17548 name: "person".to_string(),
17549 description: None,
17550 agent_instructions: None,
17551 ontology_namespace: None,
17552 ontology_class: None,
17553 }))
17554 .expect("subject entity should create");
17555 let household = block_on(service.add_entity(AddEntityRequest {
17556 domain: domain.name.as_str().into(),
17557 name: "household".to_string(),
17558 description: None,
17559 agent_instructions: None,
17560 ontology_namespace: None,
17561 ontology_class: None,
17562 }))
17563 .expect("object entity should create");
17564 let member_of = block_on(service.add_entity_relation(AddEntityRelationRequest {
17565 domain: domain.name.as_str().into(),
17566 name: "member_of".to_string(),
17567 subject: person.name.as_str().to_string(),
17568 object: household.name.as_str().to_string(),
17569 description: None,
17570 agent_instructions: None,
17571 ontology_namespace: None,
17572 ontology_class: None,
17573 }))
17574 .expect("relation definition should create");
17575 let subject = block_on(service.add_entity_instance(AddEntityInstanceRequest {
17576 domain: domain.name.as_str().to_string(),
17577 entity: person.name.as_str().to_string(),
17578 transformation_id: TransformationId(51),
17579 label: Some("Person 1".to_string()),
17580 note: None,
17581 }))
17582 .expect("subject instance should create");
17583 let object = block_on(service.add_entity_instance(AddEntityInstanceRequest {
17584 domain: domain.name.as_str().to_string(),
17585 entity: household.name.as_str().to_string(),
17586 transformation_id: TransformationId(51),
17587 label: Some("Household 1".to_string()),
17588 note: None,
17589 }))
17590 .expect("object instance should create");
17591 let relation = block_on(service.add_relation_instance(AddRelationInstanceRequest {
17592 domain: domain.name.as_str().to_string(),
17593 name: member_of.name.as_str().to_string(),
17594 subject_instance_id: subject.instance_id,
17595 object_instance_id: object.instance_id,
17596 transformation_id: TransformationId(52),
17597 valid_from: chrono::NaiveDate::from_ymd_opt(2024, 1, 1),
17598 valid_to: None,
17599 note: Some("fixture relation".to_string()),
17600 }))
17601 .expect("relation instance should create");
17602
17603 let empty_mappings = block_on(service.list_study_relation_instance_mappings(
17604 ListStudyRelationInstanceMappingsRequest {
17605 study: study.name.as_str().to_string(),
17606 domain: None,
17607 relation: None,
17608 },
17609 ))
17610 .expect("empty mapping list should succeed");
17611 assert!(empty_mappings.mappings.is_empty());
17612
17613 let mapping = block_on(service.add_study_relation_instance_mapping(
17614 AddStudyRelationInstanceMappingRequest {
17615 study: study.name.as_str().to_string(),
17616 relation_instance_id: relation.relation_instance_id,
17617 external_id: "membership-1".to_string(),
17618 transformation_id: TransformationId(53),
17619 },
17620 ))
17621 .expect("relation mapping should create");
17622 let mapping_list = block_on(service.list_study_relation_instance_mappings(
17623 ListStudyRelationInstanceMappingsRequest {
17624 study: study.name.as_str().to_string(),
17625 domain: Some(domain.name.as_str().to_string()),
17626 relation: Some(member_of.name.as_str().to_string()),
17627 },
17628 ))
17629 .expect("filtered relation mappings should list");
17630 assert_eq!(mapping_list.mappings.len(), 1);
17631 assert_eq!(mapping_list.mappings[0].mapping, mapping.mapping);
17632 assert_eq!(mapping_list.mappings[0].instance, relation);
17633 assert_eq!(mapping_list.mappings[0].subject_instance, subject);
17634 assert_eq!(mapping_list.mappings[0].object_instance, object);
17635
17636 let asset = test_asset(study.study_id, "memberships");
17637 let version = block_on(asset_repositories.assets.save_asset(&asset))
17638 .and_then(|asset| {
17639 block_on(
17640 asset_repositories
17641 .assets
17642 .save_asset_version(&test_asset_version(
17643 asset.asset_id,
17644 1,
17645 0,
17646 0,
17647 "memberships v1",
17648 )),
17649 )
17650 })
17651 .expect("dataset asset and version should save");
17652 asset_repositories.seed_dataset(DatasetRecord {
17653 dataset_id: version.version_id,
17654 });
17655 let subject_variable = test_variable(
17656 601,
17657 domain.domain_id.0,
17658 "person_id",
17659 KeyRole::External,
17660 None,
17661 );
17662 let object_variable = test_variable(
17663 602,
17664 domain.domain_id.0,
17665 "household_id",
17666 KeyRole::External,
17667 None,
17668 );
17669 let relation_variable = test_variable(
17670 603,
17671 domain.domain_id.0,
17672 "membership_id",
17673 KeyRole::External,
17674 None,
17675 );
17676 asset_repositories.seed_variable(subject_variable.clone());
17677 asset_repositories.seed_variable(object_variable.clone());
17678 asset_repositories.seed_variable(relation_variable.clone());
17679 for variable in [&subject_variable, &object_variable, &relation_variable] {
17680 block_on(asset_repositories.variables.link_dataset_variable(
17681 version.version_id,
17682 variable.variable_id,
17683 KeyRole::External,
17684 ))
17685 .expect("dataset variable fixture should link");
17686 }
17687
17688 let missing_precise_dataset_link = block_on(service.get_relation_dataset_link(
17689 GetRelationDatasetLinkRequest {
17690 relation_instance_id: relation.relation_instance_id,
17691 study: Some(study.name.as_str().to_string()),
17692 dataset: asset.name.as_str().to_string(),
17693 version: Some("latest".to_string()),
17694 },
17695 ))
17696 .expect("missing precise relation dataset readback should resolve selectors");
17697 assert!(missing_precise_dataset_link.is_none());
17698
17699 let asset_link = block_on(
17700 service.add_relation_asset_link(AddRelationAssetLinkRequest {
17701 study: study.name.as_str().to_string(),
17702 relation_instance_id: relation.relation_instance_id,
17703 asset: asset.name.as_str().to_string(),
17704 version: Some("latest".to_string()),
17705 transformation_id: TransformationId(54),
17706 }),
17707 )
17708 .expect("relation asset link should create");
17709 let asset_links_by_relation = block_on(service.list_relation_asset_links(
17710 ListRelationAssetLinksRequest {
17711 relation_instance_id: Some(relation.relation_instance_id),
17712 study: None,
17713 asset: None,
17714 version: None,
17715 },
17716 ))
17717 .expect("relation asset links should list by relation");
17718 assert_eq!(asset_links_by_relation.links.len(), 1);
17719 assert_eq!(asset_links_by_relation.links[0].link, asset_link);
17720 assert_eq!(
17721 asset_links_by_relation.links[0].study_mapping,
17722 Some(mapping.mapping.clone())
17723 );
17724 let asset_links_by_asset = block_on(service.list_relation_asset_links(
17725 ListRelationAssetLinksRequest {
17726 relation_instance_id: None,
17727 study: Some(study.name.as_str().to_string()),
17728 asset: Some(asset.name.as_str().to_string()),
17729 version: Some("latest".to_string()),
17730 },
17731 ))
17732 .expect("relation asset links should list by asset");
17733 assert_eq!(asset_links_by_asset.resolved_asset, Some(asset.clone()));
17734 assert_eq!(asset_links_by_asset.resolved_version, Some(version.clone()));
17735 assert_eq!(asset_links_by_asset.links[0].link, asset_link);
17736
17737 let dataset_link = block_on(service.add_relation_dataset_link(
17738 AddRelationDatasetLinkRequest {
17739 study: study.name.as_str().to_string(),
17740 relation_instance_id: relation.relation_instance_id,
17741 dataset: asset.name.as_str().to_string(),
17742 subject_variable: subject_variable.name.as_str().to_string(),
17743 object_variable: object_variable.name.as_str().to_string(),
17744 relation_variable: Some(relation_variable.name.as_str().to_string()),
17745 version: Some("latest".to_string()),
17746 transformation_id: TransformationId(55),
17747 },
17748 ))
17749 .expect("relation dataset link should create");
17750 let precise_dataset_link = block_on(service.get_relation_dataset_link(
17751 GetRelationDatasetLinkRequest {
17752 relation_instance_id: relation.relation_instance_id,
17753 study: Some(study.name.as_str().to_string()),
17754 dataset: asset.name.as_str().to_string(),
17755 version: Some("latest".to_string()),
17756 },
17757 ))
17758 .expect("precise relation dataset readback should succeed")
17759 .expect("precise relation dataset link should exist");
17760 assert_eq!(precise_dataset_link.link, dataset_link);
17761 assert_eq!(precise_dataset_link.subject_variable, subject_variable);
17762 assert_eq!(precise_dataset_link.object_variable, object_variable);
17763 assert_eq!(
17764 precise_dataset_link.relation_variable,
17765 Some(relation_variable)
17766 );
17767 let dataset_links = block_on(service.list_relation_dataset_links(
17768 ListRelationDatasetLinksRequest {
17769 study: Some(study.name.as_str().to_string()),
17770 dataset: asset.name.as_str().to_string(),
17771 version: Some("latest".to_string()),
17772 },
17773 ))
17774 .expect("relation dataset links should list by dataset");
17775 assert_eq!(dataset_links.resolved_dataset, Some(asset));
17776 assert_eq!(dataset_links.resolved_version, Some(version));
17777 assert_eq!(dataset_links.links.len(), 1);
17778 assert_eq!(dataset_links.links[0].link, dataset_link);
17779 }
17780
17781 #[test]
17782 fn entity_batch_from_dataset_uses_one_transformation_for_created_records() {
17783 let entity_repositories = EntityGraphRepositories::new();
17784 let asset_repositories = AssetWorkflowRepositories::new();
17785 let study = test_study("entity_batch_study");
17786 let domain = test_domain(90_210, "entity_batch_domain");
17787 entity_repositories.seed_study(study.clone());
17788 entity_repositories.seed_domain(domain.clone());
17789 asset_repositories.seed_study(study.clone());
17790 asset_repositories.seed_domain(domain.clone());
17791
17792 let service = AppService::new(AppRepositories {
17793 registrations: Arc::new(NoopCatalogueRegistrationRepository),
17794 domains: entity_repositories.domains.clone(),
17795 studies: entity_repositories.studies.clone(),
17796 study_domains: Arc::new(NoopStudyDomainRepository),
17797 study_governance: Arc::new(NoopStudyGovernanceRepository),
17798 assets: asset_repositories.assets.clone(),
17799 variables: asset_repositories.variables.clone(),
17800 vocabularies: asset_repositories.vocabularies.clone(),
17801 entities: entity_repositories.entities.clone(),
17802 transformations: asset_repositories.transformations.clone(),
17803 tags: Arc::new(NoopTagRepository),
17804 });
17805
17806 let entity = block_on(service.add_entity(AddEntityRequest {
17807 domain: domain.name.as_str().into(),
17808 name: "participant".to_string(),
17809 description: None,
17810 agent_instructions: None,
17811 ontology_namespace: None,
17812 ontology_class: None,
17813 }))
17814 .expect("entity definition should be created");
17815
17816 let asset = test_asset(study.study_id, "batch_participants");
17817 let version = block_on(asset_repositories.assets.save_asset(&asset))
17818 .and_then(|asset| {
17819 block_on(
17820 asset_repositories
17821 .assets
17822 .save_asset_version(&test_asset_version(
17823 asset.asset_id,
17824 1,
17825 0,
17826 0,
17827 "batch participants v1",
17828 )),
17829 )
17830 })
17831 .expect("dataset asset and version should save");
17832 asset_repositories.seed_dataset(DatasetRecord {
17833 dataset_id: version.version_id,
17834 });
17835 let participant_id = test_variable(
17836 701,
17837 domain.domain_id.0,
17838 "participant_id",
17839 KeyRole::External,
17840 None,
17841 );
17842 let participant_name = test_variable(
17843 702,
17844 domain.domain_id.0,
17845 "participant_name",
17846 KeyRole::None,
17847 None,
17848 );
17849 asset_repositories.seed_variable(participant_id.clone());
17850 asset_repositories.seed_variable(participant_name.clone());
17851 block_on(asset_repositories.variables.link_dataset_variable(
17852 version.version_id,
17853 participant_id.variable_id,
17854 KeyRole::External,
17855 ))
17856 .expect("external id variable should link to dataset");
17857 block_on(asset_repositories.variables.link_dataset_variable(
17858 version.version_id,
17859 participant_name.variable_id,
17860 KeyRole::None,
17861 ))
17862 .expect("label variable should link to dataset");
17863
17864 let lake_root = temp_lake_root_path();
17865 let session = test_lake_session(&lake_root);
17866 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
17867 study.study_id,
17868 asset.name.as_str(),
17869 version.major,
17870 version.minor,
17871 version.patch,
17872 );
17873 session
17874 .lake_connection()
17875 .execute(
17876 &format!(
17877 "CREATE SCHEMA IF NOT EXISTS {}",
17878 quote_identifier(&relation.schema_name)
17879 ),
17880 [],
17881 )
17882 .expect("dataset schema should be created");
17883 session
17884 .lake_connection()
17885 .execute(
17886 &format!(
17887 "CREATE TABLE {} (participant_id VARCHAR, participant_name VARCHAR)",
17888 dataset_relation_sql(&relation)
17889 ),
17890 [],
17891 )
17892 .expect("dataset table should be created");
17893 session
17894 .lake_connection()
17895 .execute(
17896 &format!(
17897 "INSERT INTO {} VALUES ('p1', 'Alice'), ('p2', 'Bob'), ('p1', 'Alice Again')",
17898 dataset_relation_sql(&relation)
17899 ),
17900 [],
17901 )
17902 .expect("dataset rows should be inserted");
17903
17904 let result = block_on(service.ensure_entity_instances_from_dataset(
17905 &session,
17906 EnsureEntityInstancesFromDatasetRequest {
17907 study: study.name.as_str().to_string(),
17908 domain: domain.name.as_str().to_string(),
17909 entity: entity.name.as_str().to_string(),
17910 dataset: asset.name.as_str().to_string(),
17911 version: None,
17912 external_id_variable: participant_id.name.as_str().to_string(),
17913 label_template: Some("Participant {participant_name}".to_string()),
17914 dry_run: false,
17915 },
17916 ))
17917 .expect("entity batch should succeed");
17918
17919 let transformation_id = result
17920 .transformation_id
17921 .expect("real batch should report transformation id");
17922 assert_eq!(result.resolved_version, "1.0.0");
17923 assert_eq!(result.counts.created_instances, 2);
17924 assert_eq!(result.counts.created_study_mappings, 2);
17925 assert_eq!(result.counts.created_dataset_links, 2);
17926
17927 let transformations = asset_repositories
17928 .transformations
17929 .state
17930 .transformations
17931 .lock()
17932 .expect("transformation lock should not be poisoned");
17933 assert_eq!(transformations.len(), 1);
17934 assert_eq!(
17935 transformations
17936 .get(&transformation_id.0)
17937 .expect("batch transformation should persist")
17938 .transformation_type,
17939 TransformationType::Entity
17940 );
17941 drop(transformations);
17942
17943 let instances = entity_repositories
17944 .state
17945 .entity_instances_by_id
17946 .lock()
17947 .expect("entity instance lock should not be poisoned");
17948 assert_eq!(instances.len(), 2);
17949 assert!(
17950 instances
17951 .values()
17952 .all(|instance| instance.transformation_id == transformation_id)
17953 );
17954 drop(instances);
17955
17956 let mappings = entity_repositories
17957 .state
17958 .study_entity_instances
17959 .lock()
17960 .expect("study entity mapping lock should not be poisoned");
17961 assert_eq!(mappings.len(), 2);
17962 assert!(
17963 mappings
17964 .iter()
17965 .all(|mapping| mapping.transformation_id == transformation_id)
17966 );
17967 drop(mappings);
17968
17969 let links = entity_repositories
17970 .state
17971 .dataset_entity_links
17972 .lock()
17973 .expect("dataset entity link lock should not be poisoned");
17974 assert_eq!(links.len(), 2);
17975 assert!(links.iter().all(|link| {
17976 link.version_id == version.version_id
17977 && link.entity_variable_id == participant_id.variable_id
17978 && link.transformation_id == transformation_id
17979 }));
17980
17981 let _ = fs::remove_dir_all(&lake_root);
17982 }
17983
17984 #[test]
17985 fn relation_batch_from_dataset_uses_one_transformation_for_created_records() {
17986 let entity_repositories = EntityGraphRepositories::new();
17987 let asset_repositories = AssetWorkflowRepositories::new();
17988 let study = test_study("relation_batch_study");
17989 let domain = test_domain(90_211, "relation_batch_domain");
17990 entity_repositories.seed_study(study.clone());
17991 entity_repositories.seed_domain(domain.clone());
17992 asset_repositories.seed_study(study.clone());
17993 asset_repositories.seed_domain(domain.clone());
17994
17995 let service = AppService::new(AppRepositories {
17996 registrations: Arc::new(NoopCatalogueRegistrationRepository),
17997 domains: entity_repositories.domains.clone(),
17998 studies: entity_repositories.studies.clone(),
17999 study_domains: Arc::new(NoopStudyDomainRepository),
18000 study_governance: Arc::new(NoopStudyGovernanceRepository),
18001 assets: asset_repositories.assets.clone(),
18002 variables: asset_repositories.variables.clone(),
18003 vocabularies: asset_repositories.vocabularies.clone(),
18004 entities: entity_repositories.entities.clone(),
18005 transformations: asset_repositories.transformations.clone(),
18006 tags: Arc::new(NoopTagRepository),
18007 });
18008
18009 let participant = block_on(service.add_entity(AddEntityRequest {
18010 domain: domain.name.as_str().into(),
18011 name: "participant".to_string(),
18012 description: None,
18013 agent_instructions: None,
18014 ontology_namespace: None,
18015 ontology_class: None,
18016 }))
18017 .expect("participant entity definition should be created");
18018 let household = block_on(service.add_entity(AddEntityRequest {
18019 domain: domain.name.as_str().into(),
18020 name: "household".to_string(),
18021 description: None,
18022 agent_instructions: None,
18023 ontology_namespace: None,
18024 ontology_class: None,
18025 }))
18026 .expect("household entity definition should be created");
18027 let membership = block_on(service.add_entity_relation(AddEntityRelationRequest {
18028 domain: domain.name.as_str().into(),
18029 name: "member_of".to_string(),
18030 subject: participant.name.as_str().to_string(),
18031 object: household.name.as_str().to_string(),
18032 description: None,
18033 agent_instructions: None,
18034 ontology_namespace: None,
18035 ontology_class: None,
18036 }))
18037 .expect("membership relation definition should be created");
18038
18039 for external_id in ["p1", "p2"] {
18040 block_on(
18041 service.ensure_study_entity_instance(EnsureStudyEntityInstanceRequest {
18042 study_id: study.study_id,
18043 entity_id: participant.entity_id,
18044 external_id: external_id.to_string(),
18045 canonical_uuid: None,
18046 label: None,
18047 note: None,
18048 transformation_id: TransformationId(7),
18049 }),
18050 )
18051 .expect("participant endpoint mapping should exist");
18052 }
18053 for external_id in ["h1", "h2"] {
18054 block_on(
18055 service.ensure_study_entity_instance(EnsureStudyEntityInstanceRequest {
18056 study_id: study.study_id,
18057 entity_id: household.entity_id,
18058 external_id: external_id.to_string(),
18059 canonical_uuid: None,
18060 label: None,
18061 note: None,
18062 transformation_id: TransformationId(7),
18063 }),
18064 )
18065 .expect("household endpoint mapping should exist");
18066 }
18067
18068 let asset = test_asset(study.study_id, "batch_memberships");
18069 let version = block_on(asset_repositories.assets.save_asset(&asset))
18070 .and_then(|asset| {
18071 block_on(
18072 asset_repositories
18073 .assets
18074 .save_asset_version(&test_asset_version(
18075 asset.asset_id,
18076 1,
18077 0,
18078 0,
18079 "batch memberships v1",
18080 )),
18081 )
18082 })
18083 .expect("dataset asset and version should save");
18084 asset_repositories.seed_dataset(DatasetRecord {
18085 dataset_id: version.version_id,
18086 });
18087 let membership_id = test_variable(
18088 711,
18089 domain.domain_id.0,
18090 "membership_id",
18091 KeyRole::External,
18092 None,
18093 );
18094 let participant_id = test_variable(
18095 712,
18096 domain.domain_id.0,
18097 "participant_id",
18098 KeyRole::External,
18099 None,
18100 );
18101 let household_id = test_variable(
18102 713,
18103 domain.domain_id.0,
18104 "household_id",
18105 KeyRole::External,
18106 None,
18107 );
18108 let start_date = test_variable(714, domain.domain_id.0, "start_date", KeyRole::None, None);
18109 for variable in [
18110 membership_id.clone(),
18111 participant_id.clone(),
18112 household_id.clone(),
18113 start_date.clone(),
18114 ] {
18115 asset_repositories.seed_variable(variable.clone());
18116 block_on(asset_repositories.variables.link_dataset_variable(
18117 version.version_id,
18118 variable.variable_id,
18119 variable.key_role,
18120 ))
18121 .expect("dataset variable should link");
18122 }
18123
18124 let lake_root = temp_lake_root_path();
18125 let session = test_lake_session(&lake_root);
18126 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
18127 study.study_id,
18128 asset.name.as_str(),
18129 version.major,
18130 version.minor,
18131 version.patch,
18132 );
18133 session
18134 .lake_connection()
18135 .execute(
18136 &format!(
18137 "CREATE SCHEMA IF NOT EXISTS {}",
18138 quote_identifier(&relation.schema_name)
18139 ),
18140 [],
18141 )
18142 .expect("dataset schema should be created");
18143 session
18144 .lake_connection()
18145 .execute(
18146 &format!(
18147 "CREATE TABLE {} (membership_id VARCHAR, participant_id VARCHAR, household_id VARCHAR, start_date VARCHAR)",
18148 dataset_relation_sql(&relation)
18149 ),
18150 [],
18151 )
18152 .expect("dataset table should be created");
18153 session
18154 .lake_connection()
18155 .execute(
18156 &format!(
18157 "INSERT INTO {} VALUES ('m1', 'p1', 'h1', '2020-01-01'), ('m2', 'p2', 'h2', '2021-02-03'), ('m1', 'p1', 'h1', '2020-01-01')",
18158 dataset_relation_sql(&relation)
18159 ),
18160 [],
18161 )
18162 .expect("dataset rows should be inserted");
18163
18164 for index in 0..12 {
18165 session
18166 .lake_connection()
18167 .execute(
18168 &format!(
18169 "INSERT INTO {} VALUES ('invalid{}','p1','h1','invalid-date')",
18170 dataset_relation_sql(&relation),
18171 index
18172 ),
18173 [],
18174 )
18175 .unwrap();
18176 }
18177
18178 let result = block_on(service.ensure_relation_instances_from_dataset(
18179 &session,
18180 EnsureRelationInstancesFromDatasetRequest {
18181 study: study.name.as_str().to_string(),
18182 domain: domain.name.as_str().to_string(),
18183 relation: membership.name.as_str().to_string(),
18184 dataset: asset.name.as_str().to_string(),
18185 version: None,
18186 relation_external_id_variable: membership_id.name.as_str().to_string(),
18187 subject_external_id_variable: participant_id.name.as_str().to_string(),
18188 object_external_id_variable: household_id.name.as_str().to_string(),
18189 valid_from_variable: Some(start_date.name.as_str().to_string()),
18190 valid_to_variable: None,
18191 dry_run: false,
18192 },
18193 ))
18194 .expect("relation batch should succeed");
18195
18196 let transformation_id = result
18197 .transformation_id
18198 .expect("real batch should report transformation id");
18199 assert_eq!(result.resolved_version, "1.0.0");
18200 assert_eq!(result.counts.rejected_rows, 12);
18201 assert_eq!(result.rejected_rows.len(), 10);
18202 assert_eq!(result.counts.created_instances, 2);
18203 assert_eq!(result.counts.created_study_mappings, 2);
18204 assert_eq!(result.counts.created_dataset_links, 2);
18205 assert_eq!(result.counts.missing_endpoints, 0);
18206
18207 let transformations = asset_repositories
18208 .transformations
18209 .state
18210 .transformations
18211 .lock()
18212 .expect("transformation lock should not be poisoned");
18213 assert_eq!(transformations.len(), 1);
18214 assert_eq!(
18215 transformations
18216 .get(&transformation_id.0)
18217 .expect("batch transformation should persist")
18218 .transformation_type,
18219 TransformationType::Entity
18220 );
18221 drop(transformations);
18222
18223 let instances = entity_repositories
18224 .state
18225 .relation_instances_by_id
18226 .lock()
18227 .expect("relation instance lock should not be poisoned");
18228 assert_eq!(instances.len(), 2);
18229 assert!(
18230 instances
18231 .values()
18232 .all(|instance| instance.transformation_id == transformation_id)
18233 );
18234 drop(instances);
18235
18236 let mappings = entity_repositories
18237 .state
18238 .study_relation_instances
18239 .lock()
18240 .expect("study relation mapping lock should not be poisoned");
18241 assert_eq!(mappings.len(), 2);
18242 assert!(
18243 mappings
18244 .iter()
18245 .all(|mapping| mapping.transformation_id == transformation_id)
18246 );
18247 drop(mappings);
18248
18249 let links = entity_repositories
18250 .state
18251 .dataset_relation_links
18252 .lock()
18253 .expect("dataset relation link lock should not be poisoned");
18254 assert_eq!(links.len(), 2);
18255 assert!(links.iter().all(|link| {
18256 link.version_id == version.version_id
18257 && link.subject_variable_id == participant_id.variable_id
18258 && link.object_variable_id == household_id.variable_id
18259 && link.relation_variable_id == Some(membership_id.variable_id)
18260 && link.transformation_id == transformation_id
18261 }));
18262
18263 let _ = fs::remove_dir_all(&lake_root);
18264 }
18265
18266 #[test]
18267 fn create_asset_requires_existing_study() {
18268 let repositories = AssetWorkflowRepositories::new();
18269 let service = repositories.service();
18270 let asset = test_asset(StudyId(uuid::Uuid::new_v4()), "p1_11_missing_study_asset");
18271
18272 let error = block_on(service.create_asset(CreateAssetRequest { asset }))
18273 .expect_err("asset without a study should fail");
18274
18275 match error {
18276 AppError::Validation(message) => assert!(message.contains("asset study not found")),
18277 other => panic!("expected validation error, got {other:?}"),
18278 }
18279 }
18280
18281 #[test]
18282 fn create_asset_adds_asset_without_versions() {
18283 let repositories = AssetWorkflowRepositories::new();
18284 let service = repositories.service();
18285 let study = test_study("p1_11_asset_study");
18286 repositories.seed_study(study.clone());
18287 let asset = test_asset(study.study_id, "p1_11_dataset_asset");
18288
18289 let catalog = block_on(service.create_asset(CreateAssetRequest {
18290 asset: asset.clone(),
18291 }))
18292 .expect("asset creation should succeed");
18293
18294 assert_eq!(catalog.asset, asset.clone());
18295 assert!(catalog.versions.is_empty());
18296 assert_eq!(catalog.latest_version, None);
18297 assert_eq!(
18298 block_on(service.list_study_assets(ListStudyAssetsRequest {
18299 study_id: study.study_id
18300 }))
18301 .expect("study assets should list"),
18302 vec![asset]
18303 );
18304 }
18305
18306 #[test]
18307 fn asset_repository_create_new_asset_and_version_assigns_ids() {
18308 let repositories = AssetWorkflowRepositories::new();
18309 let study_id = StudyId(uuid::Uuid::new_v4());
18310 let asset = block_on(repositories.assets.create_asset(&NewAssetRecord {
18311 study_id,
18312 name: NcName::parse("assigned_dataset").unwrap(),
18313 description: Some("created without caller asset id".to_string()),
18314 agent_instructions: None,
18315 asset_type: AssetType::Dataset,
18316 date_created: None,
18317 created_by: Some("tester".to_string()),
18318 }))
18319 .expect("new asset should persist with an assigned id");
18320
18321 assert_ne!(asset.asset_id.0, uuid::Uuid::nil());
18322 assert_eq!(asset.study_id, study_id);
18323 assert_eq!(asset.name.as_str(), "assigned_dataset");
18324
18325 let version = block_on(
18326 repositories
18327 .assets
18328 .create_asset_version(&NewAssetVersionRecord {
18329 classification: ahri_tre_types::IngestClassification::derived_high(),
18330 asset_id: asset.asset_id,
18331 major: 1,
18332 minor: 0,
18333 patch: 0,
18334 version_label: None,
18335 version_note: Some("created without caller version id".to_string()),
18336 is_latest: Some(true),
18337 doi: None,
18338 created_at: None,
18339 created_by: Some("tester".to_string()),
18340 }),
18341 )
18342 .expect("new asset version should persist with an assigned id");
18343
18344 assert_ne!(version.version_id.0, uuid::Uuid::nil());
18345 assert_eq!(version.asset_id, asset.asset_id);
18346 assert_eq!(version.version_label.as_deref(), Some("v1.0.0"));
18347 assert_eq!(
18348 block_on(repositories.assets.get_asset_version(version.version_id))
18349 .expect("version lookup should succeed")
18350 .map(|value| value.version_id),
18351 Some(version.version_id)
18352 );
18353 }
18354
18355 #[test]
18356 fn stateful_asset_study_repository_search_matches_text_domain_and_tag_predicates() {
18357 let repositories = AssetWorkflowRepositories::new();
18358 let service = repositories.service();
18359 let hdss = test_domain(900, "HDSS");
18360 let apcc = test_domain(901, "APCC");
18361 let mut alpha = test_study("AlphaStudy");
18362 alpha.documentation = Some("Sentinel cohort".to_string());
18363 alpha.description = Some("Longitudinal participant discovery".to_string());
18364 let mut beta = test_study("BetaStudy");
18365 beta.documentation = Some("Comparison cohort".to_string());
18366 beta.description = Some("Secondary discovery study".to_string());
18367 repositories.seed_study(alpha.clone());
18368 repositories.seed_study(beta);
18369 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18370 repositories.seed_study_domain(alpha.study_id, apcc);
18371
18372 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18373 name: "curated".to_string(),
18374 }))
18375 .expect("curated tag should persist");
18376 block_on(
18377 service
18378 .tags
18379 .attach_tag(curated.tag_id, TagAttachmentTarget::Study(alpha.study_id)),
18380 )
18381 .expect("tag should attach to study");
18382
18383 let page = block_on(repositories.studies.search_studies(
18384 &ahri_tre_core::StudySearchQuery {
18385 kind: ahri_tre_core::PredicateSetKind::AllOf,
18386 predicates: vec![
18387 ahri_tre_core::StudySearchPredicate::Text {
18388 field: ahri_tre_core::StudySearchTextField::Title,
18389 mode: ahri_tre_core::SearchTextMode::Contains,
18390 value: "sentinel".to_string(),
18391 },
18392 ahri_tre_core::StudySearchPredicate::Domain(
18393 ahri_tre_core::StudySearchDomainSelector::Name(
18394 hdss.name.as_str().to_string(),
18395 ),
18396 ),
18397 ahri_tre_core::StudySearchPredicate::Tag(
18398 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18399 ),
18400 ],
18401 limit: 10,
18402 after: None,
18403 },
18404 ))
18405 .expect("search should succeed");
18406
18407 assert_eq!(page.items.len(), 1);
18408 assert_eq!(page.items[0].study_id, alpha.study_id);
18409 assert!(page.next_cursor.is_none());
18410 }
18411
18412 #[test]
18413 fn search_studies_returns_ordered_pages_and_bounded_tags() {
18414 let repositories = AssetWorkflowRepositories::new();
18415 let service = repositories.service();
18416 let hdss = test_domain(902, "HDSS");
18417 let mut alpha = test_study("AlphaStudy");
18418 alpha.documentation = Some("Sentinel cohort alpha".to_string());
18419 alpha.description = Some("First study description".to_string());
18420 let mut beta = test_study("BetaStudy");
18421 beta.documentation = Some("Sentinel cohort beta".to_string());
18422 beta.description = Some("Second study description".to_string());
18423 repositories.seed_study(alpha.clone());
18424 repositories.seed_study(beta.clone());
18425 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18426 repositories.seed_study_domain(beta.study_id, hdss.clone());
18427
18428 for index in 0..(MAX_SUMMARY_TAGS + 2) {
18429 let tag = block_on(service.tags.create_tag(&NewTagRecord {
18430 name: format!("tag{index:02}"),
18431 }))
18432 .expect("tag should persist");
18433 block_on(
18434 service
18435 .tags
18436 .attach_tag(tag.tag_id, TagAttachmentTarget::Study(alpha.study_id)),
18437 )
18438 .expect("tag should attach");
18439 }
18440
18441 let listed = block_on(service.list_studies(ListStudiesRequest {}))
18442 .expect("study list should include tags");
18443 let fetched = block_on(service.get_study(GetStudyRequest {
18444 study: StudySelector::Id {
18445 study_id: alpha.study_id,
18446 },
18447 }))
18448 .expect("study lookup should include tags")
18449 .expect("study should exist");
18450 assert_eq!(
18451 listed
18452 .studies
18453 .iter()
18454 .find(|registration| registration.study.study_id == alpha.study_id)
18455 .expect("listed Study should exist")
18456 .tags
18457 .len(),
18458 MAX_SUMMARY_TAGS
18459 );
18460 assert_eq!(fetched.tags.len(), MAX_SUMMARY_TAGS);
18461
18462 let query = SearchStudiesRequest {
18463 kind: ahri_tre_core::PredicateSetKind::AllOf,
18464 predicates: vec![
18465 ahri_tre_core::StudySearchPredicate::Text {
18466 field: ahri_tre_core::StudySearchTextField::Title,
18467 mode: ahri_tre_core::SearchTextMode::Contains,
18468 value: "sentinel cohort".to_string(),
18469 },
18470 ahri_tre_core::StudySearchPredicate::Domain(
18471 ahri_tre_core::StudySearchDomainSelector::Name(hdss.name.as_str().to_string()),
18472 ),
18473 ],
18474 limit: 1,
18475 after: None,
18476 };
18477
18478 let first = block_on(service.search_studies(query.clone())).expect("first page");
18479 assert_eq!(first.studies.len(), 1);
18480 assert_eq!(first.studies[0].study.study_id, alpha.study_id);
18481 assert_eq!(first.studies[0].tags.len(), MAX_SUMMARY_TAGS);
18482 assert!(first.next_cursor.is_some());
18483
18484 let second = block_on(service.search_studies(SearchStudiesRequest {
18485 after: first.next_cursor.clone(),
18486 ..query
18487 }))
18488 .expect("second page");
18489 assert_eq!(second.studies.len(), 1);
18490 assert_eq!(second.studies[0].study.study_id, beta.study_id);
18491 assert!(second.next_cursor.is_none());
18492 }
18493
18494 #[test]
18495 fn search_studies_supports_single_tag_and_mixed_any_of_predicates() {
18496 let repositories = AssetWorkflowRepositories::new();
18497 let service = repositories.service();
18498 let hdss = test_domain(903, "HDSS");
18499 let apcc = test_domain(904, "APCC");
18500 let mut alpha = test_study("AlphaStudy");
18501 alpha.documentation = Some("Sentinel alpha".to_string());
18502 let mut beta = test_study("BetaStudy");
18503 beta.documentation = Some("Control beta".to_string());
18504 let mut gamma = test_study("GammaStudy");
18505 gamma.documentation = Some("Sentinel gamma".to_string());
18506 repositories.seed_study(alpha.clone());
18507 repositories.seed_study(beta.clone());
18508 repositories.seed_study(gamma.clone());
18509 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18510 repositories.seed_study_domain(beta.study_id, apcc.clone());
18511 repositories.seed_study_domain(gamma.study_id, apcc);
18512
18513 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18514 name: "curated".to_string(),
18515 }))
18516 .expect("curated tag should persist");
18517 let priority = block_on(service.tags.create_tag(&NewTagRecord {
18518 name: "priority".to_string(),
18519 }))
18520 .expect("priority tag should persist");
18521 block_on(
18522 service
18523 .tags
18524 .attach_tag(curated.tag_id, TagAttachmentTarget::Study(alpha.study_id)),
18525 )
18526 .expect("curated tag should attach");
18527 block_on(
18528 service
18529 .tags
18530 .attach_tag(priority.tag_id, TagAttachmentTarget::Study(beta.study_id)),
18531 )
18532 .expect("priority tag should attach");
18533
18534 let curated_only = block_on(service.search_studies(SearchStudiesRequest {
18535 kind: ahri_tre_core::PredicateSetKind::AllOf,
18536 predicates: vec![ahri_tre_core::StudySearchPredicate::Tag(
18537 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18538 )],
18539 limit: 10,
18540 after: None,
18541 }))
18542 .expect("single-tag search should succeed");
18543 assert_eq!(curated_only.studies.len(), 1);
18544 assert_eq!(curated_only.studies[0].study.study_id, alpha.study_id);
18545
18546 let mixed_any_of = block_on(service.search_studies(SearchStudiesRequest {
18547 kind: ahri_tre_core::PredicateSetKind::AnyOf,
18548 predicates: vec![
18549 ahri_tre_core::StudySearchPredicate::Text {
18550 field: ahri_tre_core::StudySearchTextField::Title,
18551 mode: ahri_tre_core::SearchTextMode::Contains,
18552 value: "sentinel".to_string(),
18553 },
18554 ahri_tre_core::StudySearchPredicate::Domain(
18555 ahri_tre_core::StudySearchDomainSelector::Name(hdss.name.as_str().to_string()),
18556 ),
18557 ahri_tre_core::StudySearchPredicate::Tag(
18558 ahri_tre_core::StudySearchTagSelector::Name("priority".to_string()),
18559 ),
18560 ],
18561 limit: 10,
18562 after: None,
18563 }))
18564 .expect("mixed any_of search should succeed");
18565
18566 assert_eq!(
18567 mixed_any_of
18568 .studies
18569 .iter()
18570 .map(|study| study.study.name.as_str())
18571 .collect::<Vec<_>>(),
18572 vec!["AlphaStudy", "BetaStudy", "GammaStudy"]
18573 );
18574 }
18575
18576 #[test]
18577 fn search_studies_tag_predicates_follow_all_of_duplicate_and_contradictory_semantics() {
18578 let repositories = AssetWorkflowRepositories::new();
18579 let service = repositories.service();
18580 let hdss = test_domain(905, "HDSS");
18581 let alpha = test_study("AlphaStudy");
18582 let beta = test_study("BetaStudy");
18583 let combo = test_study("ComboStudy");
18584 repositories.seed_study(alpha.clone());
18585 repositories.seed_study(beta.clone());
18586 repositories.seed_study(combo.clone());
18587 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18588 repositories.seed_study_domain(beta.study_id, hdss.clone());
18589 repositories.seed_study_domain(combo.study_id, hdss);
18590
18591 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18592 name: "curated".to_string(),
18593 }))
18594 .expect("curated tag should persist");
18595 let priority = block_on(service.tags.create_tag(&NewTagRecord {
18596 name: "priority".to_string(),
18597 }))
18598 .expect("priority tag should persist");
18599 let withdrawn = block_on(service.tags.create_tag(&NewTagRecord {
18600 name: "withdrawn".to_string(),
18601 }))
18602 .expect("withdrawn tag should persist");
18603
18604 for study_id in [alpha.study_id, combo.study_id] {
18605 block_on(
18606 service
18607 .tags
18608 .attach_tag(curated.tag_id, TagAttachmentTarget::Study(study_id)),
18609 )
18610 .expect("curated tag should attach");
18611 }
18612 for study_id in [beta.study_id, combo.study_id] {
18613 block_on(
18614 service
18615 .tags
18616 .attach_tag(priority.tag_id, TagAttachmentTarget::Study(study_id)),
18617 )
18618 .expect("priority tag should attach");
18619 }
18620
18621 let all_of = block_on(service.search_studies(SearchStudiesRequest {
18622 kind: ahri_tre_core::PredicateSetKind::AllOf,
18623 predicates: vec![
18624 ahri_tre_core::StudySearchPredicate::Tag(
18625 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18626 ),
18627 ahri_tre_core::StudySearchPredicate::Tag(
18628 ahri_tre_core::StudySearchTagSelector::Name("priority".to_string()),
18629 ),
18630 ],
18631 limit: 10,
18632 after: None,
18633 }))
18634 .expect("all_of tag search should succeed");
18635 assert_eq!(all_of.studies.len(), 1);
18636 assert_eq!(all_of.studies[0].study.study_id, combo.study_id);
18637
18638 let duplicate = block_on(service.search_studies(SearchStudiesRequest {
18639 kind: ahri_tre_core::PredicateSetKind::AllOf,
18640 predicates: vec![
18641 ahri_tre_core::StudySearchPredicate::Tag(
18642 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18643 ),
18644 ahri_tre_core::StudySearchPredicate::Tag(
18645 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18646 ),
18647 ],
18648 limit: 10,
18649 after: None,
18650 }))
18651 .expect("duplicate tag search should succeed");
18652 assert_eq!(
18653 duplicate
18654 .studies
18655 .iter()
18656 .map(|study| study.study.name.as_str())
18657 .collect::<Vec<_>>(),
18658 vec!["AlphaStudy", "ComboStudy"]
18659 );
18660
18661 let contradictory = block_on(service.search_studies(SearchStudiesRequest {
18662 kind: ahri_tre_core::PredicateSetKind::AllOf,
18663 predicates: vec![
18664 ahri_tre_core::StudySearchPredicate::Tag(
18665 ahri_tre_core::StudySearchTagSelector::Name("curated".to_string()),
18666 ),
18667 ahri_tre_core::StudySearchPredicate::Tag(
18668 ahri_tre_core::StudySearchTagSelector::Id(withdrawn.tag_id),
18669 ),
18670 ],
18671 limit: 10,
18672 after: None,
18673 }))
18674 .expect("contradictory tag search should succeed");
18675 assert!(contradictory.studies.is_empty());
18676 assert!(contradictory.next_cursor.is_none());
18677 }
18678
18679 #[test]
18680 fn stateful_asset_repository_search_matches_dataset_metadata_predicates() {
18681 let repositories = AssetWorkflowRepositories::new();
18682 let service = repositories.service();
18683 let hdss = test_domain(906, "HDSS");
18684 let apcc = test_domain(907, "APCC");
18685 let alpha = test_study("AlphaStudy");
18686 let beta = test_study("BetaStudy");
18687 repositories.seed_study(alpha.clone());
18688 repositories.seed_study(beta.clone());
18689 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18690 repositories.seed_study_domain(beta.study_id, apcc);
18691
18692 let mut alpha_asset = test_asset(alpha.study_id, "baseline_dataset");
18693 alpha_asset.description = Some("Sentinel baseline cohort extract".to_string());
18694 let alpha_version =
18695 seed_completed_dataset_version(&repositories, &alpha_asset, 1, 2, 0, true, "alpha");
18696 let mut beta_asset = test_asset(beta.study_id, "control_dataset");
18697 beta_asset.description = Some("Comparison registry extract".to_string());
18698 let _beta_version =
18699 seed_completed_dataset_version(&repositories, &beta_asset, 2, 0, 0, true, "beta");
18700
18701 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18702 name: "curated".to_string(),
18703 }))
18704 .expect("curated tag should persist");
18705 block_on(service.tags.attach_tag(
18706 curated.tag_id,
18707 TagAttachmentTarget::AssetVersion(alpha_version.version_id),
18708 ))
18709 .expect("tag should attach to dataset version");
18710
18711 let page = block_on(repositories.assets.search_datasets(
18712 &ahri_tre_core::DatasetSearchQuery {
18713 kind: ahri_tre_core::PredicateSetKind::AllOf,
18714 predicates: vec![
18715 ahri_tre_core::DatasetSearchPredicate::Text {
18716 field: ahri_tre_core::DatasetSearchTextField::Description,
18717 mode: ahri_tre_core::SearchTextMode::Contains,
18718 value: "baseline".to_string(),
18719 },
18720 ahri_tre_core::DatasetSearchPredicate::Domain(
18721 ahri_tre_core::DatasetSearchDomainSelector::Name(
18722 hdss.name.as_str().to_string(),
18723 ),
18724 ),
18725 ahri_tre_core::DatasetSearchPredicate::VersionLabel(
18726 alpha_version
18727 .version_label
18728 .clone()
18729 .expect("version label should be present"),
18730 ),
18731 ahri_tre_core::DatasetSearchPredicate::Tag(
18732 ahri_tre_core::DatasetSearchTagSelector::Name("curated".to_string()),
18733 ),
18734 ],
18735 limit: 10,
18736 after: None,
18737 },
18738 ))
18739 .expect("dataset search should succeed");
18740
18741 assert_eq!(page.items.len(), 1);
18742 assert_eq!(page.items[0].study_id, alpha.study_id);
18743 assert_eq!(page.items[0].asset.asset_id, alpha_asset.asset_id);
18744 assert!(page.next_cursor.is_none());
18745 }
18746
18747 #[test]
18748 fn search_datasets_returns_ordered_pages_and_bounded_tags() {
18749 let repositories = AssetWorkflowRepositories::new();
18750 let service = repositories.service();
18751 let hdss = test_domain(908, "HDSS");
18752 let alpha = test_study("AlphaStudy");
18753 let beta = test_study("BetaStudy");
18754 repositories.seed_study(alpha.clone());
18755 repositories.seed_study(beta.clone());
18756 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18757 repositories.seed_study_domain(beta.study_id, hdss);
18758
18759 let mut alpha_asset = test_asset(alpha.study_id, "alpha_dataset");
18760 alpha_asset.description = Some("Sentinel alpha dataset".to_string());
18761 let alpha_version =
18762 seed_completed_dataset_version(&repositories, &alpha_asset, 1, 0, 0, true, "alpha");
18763 let mut beta_asset = test_asset(beta.study_id, "beta_dataset");
18764 beta_asset.description = Some("Sentinel beta dataset".to_string());
18765 let _beta_version =
18766 seed_completed_dataset_version(&repositories, &beta_asset, 1, 0, 0, true, "beta");
18767
18768 for index in 0..(MAX_SUMMARY_TAGS + 2) {
18769 let tag = block_on(service.tags.create_tag(&NewTagRecord {
18770 name: format!("tag{index:02}"),
18771 }))
18772 .expect("tag should persist");
18773 block_on(service.tags.attach_tag(
18774 tag.tag_id,
18775 TagAttachmentTarget::AssetVersion(alpha_version.version_id),
18776 ))
18777 .expect("tag should attach");
18778 }
18779
18780 let query = SearchDatasetsRequest {
18781 kind: ahri_tre_core::PredicateSetKind::AllOf,
18782 predicates: vec![ahri_tre_core::DatasetSearchPredicate::Text {
18783 field: ahri_tre_core::DatasetSearchTextField::Description,
18784 mode: ahri_tre_core::SearchTextMode::Contains,
18785 value: "sentinel".to_string(),
18786 }],
18787 limit: 1,
18788 after: None,
18789 };
18790
18791 let first = block_on(service.search_datasets(query.clone())).expect("first page");
18792 assert_eq!(first.datasets.len(), 1);
18793 assert_eq!(first.datasets[0].asset.asset_id, alpha_asset.asset_id);
18794 assert_eq!(first.datasets[0].tags.len(), MAX_SUMMARY_TAGS);
18795 assert_eq!(first.datasets[0].variable_count, Some(0));
18796 assert!(first.next_cursor.is_some());
18797
18798 let second = block_on(service.search_datasets(SearchDatasetsRequest {
18799 after: first.next_cursor.clone(),
18800 ..query
18801 }))
18802 .expect("second page");
18803 assert_eq!(second.datasets.len(), 1);
18804 assert_eq!(second.datasets[0].asset.asset_id, beta_asset.asset_id);
18805 assert!(second.next_cursor.is_none());
18806 }
18807
18808 #[test]
18809 fn search_datasets_supports_version_label_study_domain_and_tag_predicates() {
18810 let repositories = AssetWorkflowRepositories::new();
18811 let service = repositories.service();
18812 let hdss = test_domain(909, "HDSS");
18813 let apcc = test_domain(910, "APCC");
18814 let alpha = test_study("AlphaStudy");
18815 let beta = test_study("BetaStudy");
18816 let gamma = test_study("GammaStudy");
18817 repositories.seed_study(alpha.clone());
18818 repositories.seed_study(beta.clone());
18819 repositories.seed_study(gamma.clone());
18820 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18821 repositories.seed_study_domain(beta.study_id, apcc.clone());
18822 repositories.seed_study_domain(gamma.study_id, apcc);
18823
18824 let mut alpha_asset = test_asset(alpha.study_id, "alpha_dataset");
18825 alpha_asset.description = Some("Sentinel alpha export".to_string());
18826 let alpha_version =
18827 seed_completed_dataset_version(&repositories, &alpha_asset, 1, 0, 0, true, "alpha");
18828 let mut beta_asset = test_asset(beta.study_id, "beta_dataset");
18829 beta_asset.description = Some("Control beta registry".to_string());
18830 let _beta_version =
18831 seed_completed_dataset_version(&repositories, &beta_asset, 2, 0, 0, true, "beta");
18832 let mut gamma_asset = test_asset(gamma.study_id, "gamma_dataset");
18833 gamma_asset.description = Some("Sentinel gamma feed".to_string());
18834 let _gamma_version =
18835 seed_completed_dataset_version(&repositories, &gamma_asset, 3, 0, 0, true, "gamma");
18836
18837 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18838 name: "curated".to_string(),
18839 }))
18840 .expect("curated tag should persist");
18841 let priority = block_on(service.tags.create_tag(&NewTagRecord {
18842 name: "priority".to_string(),
18843 }))
18844 .expect("priority tag should persist");
18845 block_on(service.tags.attach_tag(
18846 curated.tag_id,
18847 TagAttachmentTarget::AssetVersion(alpha_version.version_id),
18848 ))
18849 .expect("curated tag should attach");
18850 let beta_version_id =
18851 block_on(repositories.assets.list_asset_versions(beta_asset.asset_id))
18852 .expect("beta versions should load")[0]
18853 .version_id;
18854 block_on(service.tags.attach_tag(
18855 priority.tag_id,
18856 TagAttachmentTarget::AssetVersion(beta_version_id),
18857 ))
18858 .expect("priority tag should attach");
18859
18860 let version_label = block_on(service.search_datasets(SearchDatasetsRequest {
18861 kind: ahri_tre_core::PredicateSetKind::AllOf,
18862 predicates: vec![ahri_tre_core::DatasetSearchPredicate::VersionLabel(
18863 alpha_version
18864 .version_label
18865 .clone()
18866 .expect("version label should be present"),
18867 )],
18868 limit: 10,
18869 after: None,
18870 }))
18871 .expect("version label search should succeed");
18872 assert_eq!(version_label.datasets.len(), 1);
18873 assert_eq!(
18874 version_label.datasets[0].asset.asset_id,
18875 alpha_asset.asset_id
18876 );
18877
18878 let mixed_any_of = block_on(service.search_datasets(SearchDatasetsRequest {
18879 kind: ahri_tre_core::PredicateSetKind::AnyOf,
18880 predicates: vec![
18881 ahri_tre_core::DatasetSearchPredicate::Study(
18882 ahri_tre_core::DatasetSearchStudySelector::Name {
18883 domain: Some(ahri_tre_core::DatasetSearchDomainSelector::Name(
18884 "APCC".to_string(),
18885 )),
18886 name: beta.name.as_str().to_string(),
18887 },
18888 ),
18889 ahri_tre_core::DatasetSearchPredicate::Domain(
18890 ahri_tre_core::DatasetSearchDomainSelector::Name(
18891 hdss.name.as_str().to_string(),
18892 ),
18893 ),
18894 ahri_tre_core::DatasetSearchPredicate::Tag(
18895 ahri_tre_core::DatasetSearchTagSelector::Name("priority".to_string()),
18896 ),
18897 ],
18898 limit: 10,
18899 after: None,
18900 }))
18901 .expect("mixed any_of search should succeed");
18902
18903 assert_eq!(
18904 mixed_any_of
18905 .datasets
18906 .iter()
18907 .map(|dataset| dataset.asset.name.as_str())
18908 .collect::<Vec<_>>(),
18909 vec!["alpha_dataset", "beta_dataset"]
18910 );
18911 }
18912
18913 #[test]
18914 fn search_datasets_tag_predicates_follow_all_of_duplicate_and_contradictory_semantics() {
18915 let repositories = AssetWorkflowRepositories::new();
18916 let service = repositories.service();
18917 let hdss = test_domain(911, "HDSS");
18918 let alpha = test_study("AlphaStudy");
18919 let beta = test_study("BetaStudy");
18920 let combo = test_study("ComboStudy");
18921 repositories.seed_study(alpha.clone());
18922 repositories.seed_study(beta.clone());
18923 repositories.seed_study(combo.clone());
18924 repositories.seed_study_domain(alpha.study_id, hdss.clone());
18925 repositories.seed_study_domain(beta.study_id, hdss.clone());
18926 repositories.seed_study_domain(combo.study_id, hdss);
18927
18928 let alpha_version = seed_completed_dataset_version(
18929 &repositories,
18930 &test_asset(alpha.study_id, "alpha_dataset"),
18931 1,
18932 0,
18933 0,
18934 true,
18935 "alpha",
18936 );
18937 let beta_version = seed_completed_dataset_version(
18938 &repositories,
18939 &test_asset(beta.study_id, "beta_dataset"),
18940 1,
18941 0,
18942 0,
18943 true,
18944 "beta",
18945 );
18946 let combo_version = seed_completed_dataset_version(
18947 &repositories,
18948 &test_asset(combo.study_id, "combo_dataset"),
18949 1,
18950 0,
18951 0,
18952 true,
18953 "combo",
18954 );
18955
18956 let curated = block_on(service.tags.create_tag(&NewTagRecord {
18957 name: "curated".to_string(),
18958 }))
18959 .expect("curated tag should persist");
18960 let priority = block_on(service.tags.create_tag(&NewTagRecord {
18961 name: "priority".to_string(),
18962 }))
18963 .expect("priority tag should persist");
18964 let withdrawn = block_on(service.tags.create_tag(&NewTagRecord {
18965 name: "withdrawn".to_string(),
18966 }))
18967 .expect("withdrawn tag should persist");
18968
18969 for version_id in [alpha_version.version_id, combo_version.version_id] {
18970 block_on(service.tags.attach_tag(
18971 curated.tag_id,
18972 TagAttachmentTarget::AssetVersion(version_id),
18973 ))
18974 .expect("curated tag should attach");
18975 }
18976 for version_id in [beta_version.version_id, combo_version.version_id] {
18977 block_on(service.tags.attach_tag(
18978 priority.tag_id,
18979 TagAttachmentTarget::AssetVersion(version_id),
18980 ))
18981 .expect("priority tag should attach");
18982 }
18983
18984 let all_of = block_on(service.search_datasets(SearchDatasetsRequest {
18985 kind: ahri_tre_core::PredicateSetKind::AllOf,
18986 predicates: vec![
18987 ahri_tre_core::DatasetSearchPredicate::Tag(
18988 ahri_tre_core::DatasetSearchTagSelector::Name("curated".to_string()),
18989 ),
18990 ahri_tre_core::DatasetSearchPredicate::Tag(
18991 ahri_tre_core::DatasetSearchTagSelector::Name("priority".to_string()),
18992 ),
18993 ],
18994 limit: 10,
18995 after: None,
18996 }))
18997 .expect("all_of tag search should succeed");
18998 assert_eq!(all_of.datasets.len(), 1);
18999 assert_eq!(all_of.datasets[0].asset.name.as_str(), "combo_dataset");
19000
19001 let duplicate = block_on(service.search_datasets(SearchDatasetsRequest {
19002 kind: ahri_tre_core::PredicateSetKind::AllOf,
19003 predicates: vec![
19004 ahri_tre_core::DatasetSearchPredicate::Tag(
19005 ahri_tre_core::DatasetSearchTagSelector::Name("curated".to_string()),
19006 ),
19007 ahri_tre_core::DatasetSearchPredicate::Tag(
19008 ahri_tre_core::DatasetSearchTagSelector::Name("curated".to_string()),
19009 ),
19010 ],
19011 limit: 10,
19012 after: None,
19013 }))
19014 .expect("duplicate tag search should succeed");
19015 assert_eq!(
19016 duplicate
19017 .datasets
19018 .iter()
19019 .map(|dataset| dataset.asset.name.as_str())
19020 .collect::<Vec<_>>(),
19021 vec!["alpha_dataset", "combo_dataset"]
19022 );
19023
19024 let contradictory = block_on(service.search_datasets(SearchDatasetsRequest {
19025 kind: ahri_tre_core::PredicateSetKind::AllOf,
19026 predicates: vec![
19027 ahri_tre_core::DatasetSearchPredicate::Tag(
19028 ahri_tre_core::DatasetSearchTagSelector::Name("curated".to_string()),
19029 ),
19030 ahri_tre_core::DatasetSearchPredicate::Tag(
19031 ahri_tre_core::DatasetSearchTagSelector::Id(withdrawn.tag_id),
19032 ),
19033 ],
19034 limit: 10,
19035 after: None,
19036 }))
19037 .expect("contradictory tag search should succeed");
19038 assert!(contradictory.datasets.is_empty());
19039 assert!(contradictory.next_cursor.is_none());
19040 }
19041
19042 #[test]
19043 fn tag_repository_create_tag_assigns_id_and_uses_composite_attachments() {
19044 let repositories = AssetWorkflowRepositories::new();
19045 let service = repositories.service();
19046 let target = TagAttachmentTarget::Asset(AssetId(uuid::Uuid::new_v4()));
19047
19048 let tag = block_on(service.tags.create_tag(&NewTagRecord {
19049 name: "governed".to_string(),
19050 }))
19051 .expect("new tag should persist with an assigned id");
19052 block_on(service.tags.attach_tag(tag.tag_id, target.clone())).expect("tag should attach");
19053 block_on(service.tags.attach_tag(tag.tag_id, target.clone()))
19054 .expect("duplicate tag attach should be idempotent");
19055
19056 assert_eq!(tag.tag_id, TagId(1));
19057 assert_eq!(
19058 repositories
19059 .state
19060 .tag_links
19061 .lock()
19062 .expect("tag link lock should not be poisoned")
19063 .as_slice(),
19064 &[(target, tag.tag_id)]
19065 );
19066 }
19067
19068 #[test]
19069 fn study_duo_restriction_create_assigns_id_and_save_preserves_explicit_id() {
19070 let repositories = RegistrationRepositories::new();
19071 let study_id = StudyId(uuid::Uuid::new_v4());
19072
19073 let created = block_on(repositories.study_governance.create_study_duo_restriction(
19074 &NewStudyDuoRestrictionRecord {
19075 study_id,
19076 duo_id: "DUO:0000042".to_string(),
19077 qualifier_ontology_namespace: None,
19078 qualifier_ontology_id: None,
19079 qualifier_text: Some("repo assigned".to_string()),
19080 qualifier_date: None,
19081 qualifier_integer: None,
19082 },
19083 ))
19084 .expect("study DUO restriction should persist with an assigned id");
19085 let explicit = StudyDuoRestrictionRecord {
19086 study_duo_restriction_id: StudyDuoRestrictionId(99),
19087 study_id,
19088 duo_id: "DUO:0000007".to_string(),
19089 qualifier_ontology_namespace: None,
19090 qualifier_ontology_id: None,
19091 qualifier_text: Some("explicit replay".to_string()),
19092 qualifier_date: None,
19093 qualifier_integer: Some(12),
19094 };
19095 let saved = block_on(
19096 repositories
19097 .study_governance
19098 .save_study_duo_restriction(&explicit),
19099 )
19100 .expect("explicit study DUO restriction should save");
19101
19102 assert_eq!(created.study_duo_restriction_id, StudyDuoRestrictionId(1));
19103 assert_eq!(created.qualifier_text.as_deref(), Some("repo assigned"));
19104 assert_eq!(saved.study_duo_restriction_id, StudyDuoRestrictionId(99));
19105 assert_eq!(
19106 block_on(
19107 repositories
19108 .study_governance
19109 .list_study_duo_restrictions(study_id)
19110 )
19111 .expect("study DUO restrictions should list")
19112 .len(),
19113 2
19114 );
19115 }
19116
19117 #[test]
19118 fn study_duo_restriction_list_and_replace_are_metadata_only() {
19119 let repositories = RegistrationRepositories::new();
19120 let service = repositories.service();
19121 let study_id = StudyId(uuid::Uuid::new_v4());
19122 block_on(
19123 repositories
19124 .study_governance
19125 .grant_study_access(study_id, "orcid_0000-0000-0000-0002"),
19126 )
19127 .expect("access fixture should save");
19128 block_on(repositories.study_governance.create_study_duo_restriction(
19129 &NewStudyDuoRestrictionRecord {
19130 study_id,
19131 duo_id: "DUO:0000042".to_string(),
19132 qualifier_ontology_namespace: None,
19133 qualifier_ontology_id: None,
19134 qualifier_text: Some("old".to_string()),
19135 qualifier_date: None,
19136 qualifier_integer: None,
19137 },
19138 ))
19139 .expect("old study DUO restriction should save");
19140
19141 let replaced = block_on(service.replace_study_duo_restrictions(
19142 ReplaceStudyDuoRestrictionsRequest {
19143 study_id,
19144 restrictions: vec![crate::StudyDuoRestrictionInput {
19145 code: None,
19146 duo_id: Some(" DUO:0000007 ".to_string()),
19147 qualifier_ontology_namespace: None,
19148 qualifier_ontology_id: None,
19149 qualifier_text: Some("replacement".to_string()),
19150 qualifier_date: None,
19151 qualifier_integer: Some(6),
19152 }],
19153 },
19154 ))
19155 .expect("study DUO restrictions should replace through app service");
19156
19157 assert_eq!(replaced.study_id, study_id);
19158 assert_eq!(replaced.restrictions.len(), 1);
19159 assert_eq!(replaced.restrictions[0].duo_id, "DUO:0000007");
19160 let listed = block_on(
19161 service.list_study_duo_restrictions(ListStudyDuoRestrictionsRequest { study_id }),
19162 )
19163 .expect("study DUO restrictions should list through app service");
19164 assert_eq!(listed.restrictions, replaced.restrictions);
19165 assert_eq!(
19166 block_on(repositories.study_governance.list_access_grants(study_id))
19167 .expect("access grants should list")
19168 .len(),
19169 1,
19170 "DUO replacement must not grant or revoke study access"
19171 );
19172 }
19173
19174 #[test]
19175 fn study_duo_restriction_replace_rejects_ambiguous_terms_before_repository_call() {
19176 let repositories = RegistrationRepositories::new();
19177 let service = repositories.service();
19178 let study_id = StudyId(uuid::Uuid::new_v4());
19179
19180 let error = block_on(service.replace_study_duo_restrictions(
19181 ReplaceStudyDuoRestrictionsRequest {
19182 study_id,
19183 restrictions: vec![crate::StudyDuoRestrictionInput {
19184 code: Some("GRU".to_string()),
19185 duo_id: Some("DUO:0000042".to_string()),
19186 qualifier_ontology_namespace: None,
19187 qualifier_ontology_id: None,
19188 qualifier_text: None,
19189 qualifier_date: None,
19190 qualifier_integer: None,
19191 }],
19192 },
19193 ))
19194 .expect_err("ambiguous DUO restriction should fail validation");
19195
19196 assert!(matches!(error, AppError::Validation(_)));
19197 assert!(
19198 block_on(
19199 repositories
19200 .study_governance
19201 .list_study_duo_restrictions(study_id)
19202 )
19203 .expect("study DUO restrictions should list")
19204 .is_empty(),
19205 "validation should fail before the repository is called"
19206 );
19207 }
19208
19209 #[test]
19210 fn asset_version_duo_restriction_create_assigns_id_and_save_preserves_explicit_id() {
19211 let repositories = AssetWorkflowRepositories::new();
19212 let version_id = VersionId(uuid::Uuid::new_v4());
19213
19214 let created = block_on(repositories.assets.create_asset_version_duo_restriction(
19215 &NewAssetVersionDuoRestrictionRecord {
19216 version_id,
19217 duo_id: "DUO:0000042".to_string(),
19218 qualifier_ontology_namespace: None,
19219 qualifier_ontology_id: None,
19220 qualifier_text: Some("version assigned".to_string()),
19221 qualifier_date: None,
19222 qualifier_integer: None,
19223 },
19224 ))
19225 .expect("asset-version DUO restriction should persist with an assigned id");
19226 let explicit = AssetVersionDuoRestrictionRecord {
19227 asset_version_duo_restriction_id: AssetVersionDuoRestrictionId(77),
19228 version_id,
19229 duo_id: "DUO:0000007".to_string(),
19230 qualifier_ontology_namespace: None,
19231 qualifier_ontology_id: None,
19232 qualifier_text: Some("explicit replay".to_string()),
19233 qualifier_date: None,
19234 qualifier_integer: Some(7),
19235 };
19236 let saved = block_on(
19237 repositories
19238 .assets
19239 .save_asset_version_duo_restriction(&explicit),
19240 )
19241 .expect("explicit asset-version DUO restriction should save");
19242
19243 assert_eq!(
19244 created.asset_version_duo_restriction_id,
19245 AssetVersionDuoRestrictionId(1)
19246 );
19247 assert_eq!(created.qualifier_text.as_deref(), Some("version assigned"));
19248 assert_eq!(
19249 saved.asset_version_duo_restriction_id,
19250 AssetVersionDuoRestrictionId(77)
19251 );
19252 assert_eq!(
19253 block_on(
19254 repositories
19255 .assets
19256 .list_asset_version_duo_restrictions(version_id)
19257 )
19258 .expect("asset-version DUO restrictions should list")
19259 .len(),
19260 2
19261 );
19262 }
19263
19264 #[test]
19265 fn asset_version_duo_restriction_list_returns_explicit_version_rows() {
19266 let repositories = AssetWorkflowRepositories::new();
19267 let service = repositories.service();
19268 let version_id = VersionId(uuid::Uuid::new_v4());
19269 let other_version_id = VersionId(uuid::Uuid::new_v4());
19270
19271 block_on(repositories.assets.create_asset_version_duo_restriction(
19272 &NewAssetVersionDuoRestrictionRecord {
19273 version_id,
19274 duo_id: "DUO:0000042".to_string(),
19275 qualifier_ontology_namespace: Some("MONDO".to_string()),
19276 qualifier_ontology_id: Some("MONDO:0005148".to_string()),
19277 qualifier_text: None,
19278 qualifier_date: None,
19279 qualifier_integer: None,
19280 },
19281 ))
19282 .expect("target version DUO restriction should persist");
19283 block_on(repositories.assets.create_asset_version_duo_restriction(
19284 &NewAssetVersionDuoRestrictionRecord {
19285 version_id: other_version_id,
19286 duo_id: "DUO:0000007".to_string(),
19287 qualifier_ontology_namespace: None,
19288 qualifier_ontology_id: None,
19289 qualifier_text: Some("other version".to_string()),
19290 qualifier_date: None,
19291 qualifier_integer: None,
19292 },
19293 ))
19294 .expect("other version DUO restriction should persist");
19295
19296 let listed = block_on(service.list_asset_version_duo_restrictions(
19297 ListAssetVersionDuoRestrictionsRequest { version_id },
19298 ))
19299 .expect("asset-version DUO restrictions should list");
19300
19301 assert_eq!(listed.version_id, version_id);
19302 assert_eq!(listed.restrictions.len(), 1);
19303 assert_eq!(listed.restrictions[0].duo_id, "DUO:0000042");
19304 assert_eq!(
19305 listed.restrictions[0]
19306 .qualifier_ontology_namespace
19307 .as_deref(),
19308 Some("MONDO")
19309 );
19310 }
19311
19312 #[test]
19313 fn asset_version_duo_effective_prefers_explicit_version_rows() {
19314 let repositories = AssetWorkflowRepositories::new();
19315 let service = repositories.service();
19316 let study_id = StudyId(uuid::Uuid::new_v4());
19317 let version_id = VersionId(uuid::Uuid::new_v4());
19318
19319 block_on(
19320 service.replace_study_duo_restrictions(ReplaceStudyDuoRestrictionsRequest {
19321 study_id,
19322 restrictions: vec![crate::StudyDuoRestrictionInput {
19323 code: Some("DUO:0000007".to_string()),
19324 duo_id: None,
19325 qualifier_ontology_namespace: None,
19326 qualifier_ontology_id: None,
19327 qualifier_text: Some("study default".to_string()),
19328 qualifier_date: None,
19329 qualifier_integer: None,
19330 }],
19331 }),
19332 )
19333 .expect("study defaults should persist");
19334 block_on(service.replace_asset_version_duo_restrictions(
19335 ReplaceAssetVersionDuoRestrictionsRequest {
19336 version_id,
19337 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19338 code: Some("DUO:0000042".to_string()),
19339 duo_id: None,
19340 qualifier_ontology_namespace: Some("MONDO".to_string()),
19341 qualifier_ontology_id: Some("MONDO:0005148".to_string()),
19342 qualifier_text: None,
19343 qualifier_date: None,
19344 qualifier_integer: None,
19345 }],
19346 },
19347 ))
19348 .expect("asset-version override should persist");
19349
19350 let effective = block_on(service.list_effective_asset_version_duo_restrictions(
19351 ListEffectiveAssetVersionDuoRestrictionsRequest {
19352 study_id,
19353 version_id,
19354 },
19355 ))
19356 .expect("effective asset-version DUO restrictions should list");
19357
19358 assert_eq!(effective.study_id, study_id);
19359 assert_eq!(effective.version_id, version_id);
19360 assert_eq!(effective.restrictions.len(), 1);
19361 assert_eq!(
19362 effective.restrictions[0].source,
19363 AssetDuoRestrictionSource::AssetVersion
19364 );
19365 assert_eq!(effective.restrictions[0].duo_id, "DUO:0000042");
19366 assert_eq!(
19367 effective.restrictions[0]
19368 .qualifier_ontology_namespace
19369 .as_deref(),
19370 Some("MONDO")
19371 );
19372 }
19373
19374 #[test]
19375 fn asset_version_duo_effective_falls_back_to_study_defaults_without_explicit_rows() {
19376 let repositories = AssetWorkflowRepositories::new();
19377 let service = repositories.service();
19378 let study_id = StudyId(uuid::Uuid::new_v4());
19379 let version_id = VersionId(uuid::Uuid::new_v4());
19380
19381 block_on(
19382 service.replace_study_duo_restrictions(ReplaceStudyDuoRestrictionsRequest {
19383 study_id,
19384 restrictions: vec![crate::StudyDuoRestrictionInput {
19385 code: Some("GRU".to_string()),
19386 duo_id: None,
19387 qualifier_ontology_namespace: None,
19388 qualifier_ontology_id: None,
19389 qualifier_text: Some("committee approval".to_string()),
19390 qualifier_date: None,
19391 qualifier_integer: Some(3),
19392 }],
19393 }),
19394 )
19395 .expect("study defaults should persist");
19396
19397 let effective = block_on(service.list_effective_asset_version_duo_restrictions(
19398 ListEffectiveAssetVersionDuoRestrictionsRequest {
19399 study_id,
19400 version_id,
19401 },
19402 ))
19403 .expect("effective asset-version DUO restrictions should fall back");
19404
19405 assert_eq!(effective.restrictions.len(), 1);
19406 assert_eq!(
19407 effective.restrictions[0].source,
19408 AssetDuoRestrictionSource::StudyDefault
19409 );
19410 assert_eq!(effective.restrictions[0].duo_id, "GRU");
19411 assert_eq!(
19412 effective.restrictions[0].qualifier_text.as_deref(),
19413 Some("committee approval")
19414 );
19415 assert_eq!(effective.restrictions[0].qualifier_integer, Some(3));
19416 assert!(
19417 block_on(
19418 repositories
19419 .assets
19420 .list_asset_version_duo_restrictions(version_id)
19421 )
19422 .expect("explicit rows should list")
19423 .is_empty(),
19424 "effective fallback must remain read-only"
19425 );
19426 }
19427
19428 #[test]
19429 fn asset_version_duo_effective_returns_empty_when_no_defaults_exist() {
19430 let repositories = AssetWorkflowRepositories::new();
19431 let service = repositories.service();
19432 let study_id = StudyId(uuid::Uuid::new_v4());
19433 let version_id = VersionId(uuid::Uuid::new_v4());
19434
19435 let effective = block_on(service.list_effective_asset_version_duo_restrictions(
19436 ListEffectiveAssetVersionDuoRestrictionsRequest {
19437 study_id,
19438 version_id,
19439 },
19440 ))
19441 .expect("effective asset-version DUO restrictions should list when empty");
19442
19443 assert!(effective.restrictions.is_empty());
19444 }
19445
19446 #[test]
19447 fn asset_version_duo_clear_then_effective_falls_back_to_study_defaults() {
19448 let repositories = AssetWorkflowRepositories::new();
19449 let service = repositories.service();
19450 let study_id = StudyId(uuid::Uuid::new_v4());
19451 let version_id = VersionId(uuid::Uuid::new_v4());
19452
19453 block_on(
19454 service.replace_study_duo_restrictions(ReplaceStudyDuoRestrictionsRequest {
19455 study_id,
19456 restrictions: vec![crate::StudyDuoRestrictionInput {
19457 code: Some("DUO:0000007".to_string()),
19458 duo_id: None,
19459 qualifier_ontology_namespace: None,
19460 qualifier_ontology_id: None,
19461 qualifier_text: Some("study default after clear".to_string()),
19462 qualifier_date: None,
19463 qualifier_integer: None,
19464 }],
19465 }),
19466 )
19467 .expect("study defaults should persist");
19468 block_on(service.replace_asset_version_duo_restrictions(
19469 ReplaceAssetVersionDuoRestrictionsRequest {
19470 version_id,
19471 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19472 code: Some("DUO:0000042".to_string()),
19473 duo_id: None,
19474 qualifier_ontology_namespace: None,
19475 qualifier_ontology_id: None,
19476 qualifier_text: Some("explicit before clear".to_string()),
19477 qualifier_date: None,
19478 qualifier_integer: None,
19479 }],
19480 },
19481 ))
19482 .expect("asset-version override should persist");
19483
19484 block_on(service.clear_asset_version_duo_restrictions(
19485 ClearAssetVersionDuoRestrictionsRequest { version_id },
19486 ))
19487 .expect("asset-version override should clear");
19488 let effective = block_on(service.list_effective_asset_version_duo_restrictions(
19489 ListEffectiveAssetVersionDuoRestrictionsRequest {
19490 study_id,
19491 version_id,
19492 },
19493 ))
19494 .expect("effective asset-version DUO restrictions should fall back after clear");
19495
19496 assert_eq!(effective.restrictions.len(), 1);
19497 assert_eq!(
19498 effective.restrictions[0].source,
19499 AssetDuoRestrictionSource::StudyDefault
19500 );
19501 assert_eq!(
19502 effective.restrictions[0].qualifier_text.as_deref(),
19503 Some("study default after clear")
19504 );
19505 }
19506
19507 #[test]
19508 fn asset_version_duo_restriction_replace_is_complete_and_assigns_new_ids() {
19509 let repositories = AssetWorkflowRepositories::new();
19510 let service = repositories.service();
19511 let version_id = VersionId(uuid::Uuid::new_v4());
19512 let other_version_id = VersionId(uuid::Uuid::new_v4());
19513
19514 block_on(repositories.assets.create_asset_version_duo_restriction(
19515 &NewAssetVersionDuoRestrictionRecord {
19516 version_id,
19517 duo_id: "DUO:0000042".to_string(),
19518 qualifier_ontology_namespace: None,
19519 qualifier_ontology_id: None,
19520 qualifier_text: Some("old".to_string()),
19521 qualifier_date: None,
19522 qualifier_integer: None,
19523 },
19524 ))
19525 .expect("old target version DUO restriction should persist");
19526 block_on(repositories.assets.create_asset_version_duo_restriction(
19527 &NewAssetVersionDuoRestrictionRecord {
19528 version_id: other_version_id,
19529 duo_id: "DUO:0000007".to_string(),
19530 qualifier_ontology_namespace: None,
19531 qualifier_ontology_id: None,
19532 qualifier_text: Some("other version".to_string()),
19533 qualifier_date: None,
19534 qualifier_integer: None,
19535 },
19536 ))
19537 .expect("other version DUO restriction should persist");
19538
19539 let replaced = block_on(service.replace_asset_version_duo_restrictions(
19540 ReplaceAssetVersionDuoRestrictionsRequest {
19541 version_id,
19542 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19543 code: Some("GRU".to_string()),
19544 duo_id: None,
19545 qualifier_ontology_namespace: None,
19546 qualifier_ontology_id: None,
19547 qualifier_text: Some("replacement".to_string()),
19548 qualifier_date: None,
19549 qualifier_integer: Some(12),
19550 }],
19551 },
19552 ))
19553 .expect("asset-version DUO restrictions should replace through app service");
19554
19555 assert_eq!(replaced.version_id, version_id);
19556 assert_eq!(replaced.restrictions.len(), 1);
19557 assert_eq!(replaced.restrictions[0].duo_id, "GRU");
19558 assert_ne!(
19559 replaced.restrictions[0].asset_version_duo_restriction_id,
19560 AssetVersionDuoRestrictionId(1),
19561 "replacement should persist newly assigned rows rather than reusing old ids"
19562 );
19563 let listed = block_on(service.list_asset_version_duo_restrictions(
19564 ListAssetVersionDuoRestrictionsRequest { version_id },
19565 ))
19566 .expect("asset-version DUO restrictions should list after replacement");
19567 assert_eq!(listed.restrictions, replaced.restrictions);
19568 assert_eq!(
19569 block_on(
19570 repositories
19571 .assets
19572 .list_asset_version_duo_restrictions(other_version_id)
19573 )
19574 .expect("other version restrictions should list")
19575 .len(),
19576 1,
19577 "replacement must not affect other versions"
19578 );
19579 }
19580
19581 #[test]
19582 fn asset_version_duo_restriction_replace_rejects_empty_payload() {
19583 let repositories = AssetWorkflowRepositories::new();
19584 let service = repositories.service();
19585 let version_id = VersionId(uuid::Uuid::new_v4());
19586
19587 let error = block_on(service.replace_asset_version_duo_restrictions(
19588 ReplaceAssetVersionDuoRestrictionsRequest {
19589 version_id,
19590 restrictions: Vec::new(),
19591 },
19592 ))
19593 .expect_err("empty replacement should fail validation");
19594
19595 let AppError::Validation(message) = error else {
19596 panic!("expected validation error");
19597 };
19598 assert!(message.contains("asset duo clear"));
19599 }
19600
19601 #[test]
19602 fn asset_version_duo_restriction_clear_removes_only_selected_version_rows() {
19603 let repositories = AssetWorkflowRepositories::new();
19604 let service = repositories.service();
19605 let version_id = VersionId(uuid::Uuid::new_v4());
19606 let other_version_id = VersionId(uuid::Uuid::new_v4());
19607
19608 block_on(service.replace_asset_version_duo_restrictions(
19609 ReplaceAssetVersionDuoRestrictionsRequest {
19610 version_id,
19611 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19612 code: Some("GRU".to_string()),
19613 duo_id: None,
19614 qualifier_ontology_namespace: None,
19615 qualifier_ontology_id: None,
19616 qualifier_text: Some("selected version".to_string()),
19617 qualifier_date: None,
19618 qualifier_integer: None,
19619 }],
19620 },
19621 ))
19622 .expect("selected version DUO restriction should replace");
19623 block_on(service.replace_asset_version_duo_restrictions(
19624 ReplaceAssetVersionDuoRestrictionsRequest {
19625 version_id: other_version_id,
19626 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19627 code: Some("DUO:0000007".to_string()),
19628 duo_id: None,
19629 qualifier_ontology_namespace: None,
19630 qualifier_ontology_id: None,
19631 qualifier_text: Some("other version".to_string()),
19632 qualifier_date: None,
19633 qualifier_integer: None,
19634 }],
19635 },
19636 ))
19637 .expect("other version DUO restriction should replace");
19638
19639 let cleared = block_on(service.clear_asset_version_duo_restrictions(
19640 ClearAssetVersionDuoRestrictionsRequest { version_id },
19641 ))
19642 .expect("asset-version DUO restrictions should clear through app service");
19643
19644 assert_eq!(cleared.version_id, version_id);
19645 assert!(cleared.restrictions.is_empty());
19646 assert!(
19647 block_on(service.list_asset_version_duo_restrictions(
19648 ListAssetVersionDuoRestrictionsRequest { version_id },
19649 ))
19650 .expect("selected version restrictions should list after clear")
19651 .restrictions
19652 .is_empty()
19653 );
19654 assert_eq!(
19655 block_on(service.list_asset_version_duo_restrictions(
19656 ListAssetVersionDuoRestrictionsRequest {
19657 version_id: other_version_id,
19658 },
19659 ))
19660 .expect("other version restrictions should remain")
19661 .restrictions
19662 .len(),
19663 1,
19664 "clear must not affect other versions"
19665 );
19666 }
19667
19668 #[test]
19669 fn asset_version_duo_restriction_clear_is_idempotent_when_empty() {
19670 let repositories = AssetWorkflowRepositories::new();
19671 let service = repositories.service();
19672 let version_id = VersionId(uuid::Uuid::new_v4());
19673
19674 let first = block_on(service.clear_asset_version_duo_restrictions(
19675 ClearAssetVersionDuoRestrictionsRequest { version_id },
19676 ))
19677 .expect("empty clear should succeed");
19678 let second = block_on(service.clear_asset_version_duo_restrictions(
19679 ClearAssetVersionDuoRestrictionsRequest { version_id },
19680 ))
19681 .expect("repeated empty clear should succeed");
19682
19683 assert_eq!(first.version_id, version_id);
19684 assert!(first.restrictions.is_empty());
19685 assert_eq!(second.version_id, version_id);
19686 assert!(second.restrictions.is_empty());
19687 }
19688
19689 #[test]
19690 fn asset_version_duo_restriction_replace_rejects_ambiguous_terms_before_repository_call() {
19691 let repositories = AssetWorkflowRepositories::new();
19692 let service = repositories.service();
19693 let version_id = VersionId(uuid::Uuid::new_v4());
19694
19695 let error = block_on(service.replace_asset_version_duo_restrictions(
19696 ReplaceAssetVersionDuoRestrictionsRequest {
19697 version_id,
19698 restrictions: vec![crate::AssetVersionDuoRestrictionInput {
19699 code: Some("GRU".to_string()),
19700 duo_id: Some("DUO:0000042".to_string()),
19701 qualifier_ontology_namespace: None,
19702 qualifier_ontology_id: None,
19703 qualifier_text: None,
19704 qualifier_date: None,
19705 qualifier_integer: None,
19706 }],
19707 },
19708 ))
19709 .expect_err("ambiguous DUO restriction should fail validation");
19710
19711 assert!(matches!(error, AppError::Validation(_)));
19712 assert!(
19713 block_on(
19714 repositories
19715 .assets
19716 .list_asset_version_duo_restrictions(version_id)
19717 )
19718 .expect("asset-version DUO restrictions should list")
19719 .is_empty(),
19720 "validation should fail before the repository is called"
19721 );
19722 }
19723
19724 #[test]
19725 fn create_new_asset_and_version_requests_do_not_require_ids() {
19726 let repositories = AssetWorkflowRepositories::new();
19727 let service = repositories.service();
19728 let study = test_study("p1_11_new_asset_study");
19729 repositories.seed_study(study.clone());
19730
19731 let catalog = block_on(service.create_new_asset(CreateNewAssetRequest {
19732 asset: NewAssetRecord {
19733 study_id: study.study_id,
19734 name: NcName::parse("p1_11_new_dataset_asset").unwrap(),
19735 description: Some("Dataset created without caller id".to_string()),
19736 agent_instructions: None,
19737 asset_type: AssetType::Dataset,
19738 date_created: None,
19739 created_by: Some("tester".to_string()),
19740 },
19741 }))
19742 .expect("idless asset request should succeed");
19743 let asset_id = catalog.asset.asset_id;
19744
19745 let catalog = block_on(
19746 service.create_new_asset_version(CreateNewAssetVersionRequest {
19747 version: NewAssetVersionRecord {
19748 classification: ahri_tre_types::IngestClassification::derived_high(),
19749 asset_id,
19750 major: 1,
19751 minor: 0,
19752 patch: 0,
19753 version_label: None,
19754 version_note: Some("Version created without caller id".to_string()),
19755 is_latest: None,
19756 doi: None,
19757 created_at: None,
19758 created_by: Some("tester".to_string()),
19759 },
19760 }),
19761 )
19762 .expect("idless asset-version request should succeed");
19763
19764 let version = catalog
19765 .latest_version
19766 .expect("new version should become latest");
19767 assert_eq!(version.asset_id, asset_id);
19768 assert_ne!(version.version_id.0, uuid::Uuid::nil());
19769 }
19770
19771 #[test]
19772 fn asset_repository_save_preserves_explicit_asset_and_version_ids() {
19773 let repositories = AssetWorkflowRepositories::new();
19774 let asset_id = AssetId(uuid::Uuid::new_v4());
19775 let version_id = VersionId(uuid::Uuid::new_v4());
19776 let asset = AssetRecord {
19777 asset_id,
19778 study_id: StudyId(uuid::Uuid::new_v4()),
19779 name: NcName::parse("explicit_dataset").unwrap(),
19780 description: None,
19781 agent_instructions: None,
19782 asset_type: AssetType::Dataset,
19783 date_created: None,
19784 created_by: Some("fixture".to_string()),
19785 };
19786 let version = AssetVersionRecord {
19787 version_id,
19788 asset_id,
19789 major: 1,
19790 minor: 0,
19791 patch: 0,
19792 version_label: None,
19793 version_note: Some("fixture id".to_string()),
19794 is_latest: Some(true),
19795 doi: None,
19796 created_at: None,
19797 created_by: Some("fixture".to_string()),
19798 };
19799
19800 let saved_asset =
19801 block_on(repositories.assets.save_asset(&asset)).expect("explicit asset should save");
19802 let saved_version = block_on(repositories.assets.save_asset_version(&version))
19803 .expect("explicit asset version should save");
19804
19805 assert_eq!(saved_asset.asset_id, asset_id);
19806 assert_eq!(saved_version.version_id, version_id);
19807 assert_eq!(saved_version.asset_id, asset_id);
19808 }
19809
19810 #[test]
19811 fn withdraw_latest_dataset_version_promotes_previous_complete_version() {
19812 let repositories = AssetWorkflowRepositories::new();
19813 let service = repositories.service();
19814 let study = test_study("issue05_withdraw_latest_study");
19815 repositories.seed_study(study.clone());
19816 repositories.grant_study_custodianship(study.study_id, "curator@example.test");
19817 let asset = test_asset(study.study_id, "issue05_withdraw_latest_dataset");
19818 let first =
19819 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, false, "baseline");
19820 let second =
19821 seed_completed_dataset_version(&repositories, &asset, 1, 1, 0, true, "faulty refresh");
19822
19823 let result = block_on(service.withdraw_dataset_version(
19824 None,
19825 WithdrawDatasetVersionRequest {
19826 study_id: study.study_id,
19827 dataset_name: asset.name.clone(),
19828 version: "1.1.0".to_string(),
19829 reason: "bad source extract".to_string(),
19830 actor: Some("curator@example.test".to_string()),
19831 force: true,
19832 drop_lake_table: false,
19833 },
19834 ))
19835 .expect("withdrawal should succeed");
19836
19837 assert_eq!(result.withdrawn_version.version_id, second.version_id);
19838 assert_eq!(
19839 result
19840 .promoted_latest_version
19841 .as_ref()
19842 .map(|v| v.version_id),
19843 Some(first.version_id)
19844 );
19845 assert_eq!(
19846 result.catalog.latest_version.as_ref().map(|v| v.version_id),
19847 Some(first.version_id)
19848 );
19849 assert_eq!(result.withdrawal.reason, "bad source extract");
19850 assert_eq!(result.withdrawal.tombstone_state, "withdrawn");
19851 assert!(
19852 result
19853 .withdrawal
19854 .provenance
19855 .as_deref()
19856 .unwrap_or("")
19857 .contains("issue05_withdraw_latest_dataset")
19858 );
19859 }
19860
19861 #[test]
19862 fn withdraw_only_dataset_version_leaves_no_latest_and_is_queryable() {
19863 let repositories = AssetWorkflowRepositories::new();
19864 let service = repositories.service();
19865 let study = test_study("issue05_withdraw_only_study");
19866 repositories.seed_study(study.clone());
19867 repositories.grant_study_custodianship(study.study_id, "curator@example.test");
19868 let asset = test_asset(study.study_id, "issue05_withdraw_only_dataset");
19869 let only =
19870 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "only version");
19871
19872 let result = block_on(service.withdraw_dataset_version(
19873 None,
19874 WithdrawDatasetVersionRequest {
19875 study_id: study.study_id,
19876 dataset_name: asset.name.clone(),
19877 version: "1.0.0".to_string(),
19878 reason: "contains invalid rows".to_string(),
19879 actor: Some("curator@example.test".to_string()),
19880 force: true,
19881 drop_lake_table: false,
19882 },
19883 ))
19884 .expect("withdrawal should succeed");
19885
19886 assert_eq!(result.withdrawn_version.version_id, only.version_id);
19887 assert_eq!(result.promoted_latest_version, None);
19888 assert_eq!(result.catalog.latest_version, None);
19889 assert!(result.catalog.versions.is_empty());
19890 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
19891 study_id: study.study_id,
19892 include_versions: false,
19893 }))
19894 .expect("dataset list should remain readable");
19895 assert!(
19896 list.datasets.is_empty(),
19897 "dataset list should not advertise withdrawn-only datasets"
19898 );
19899 let withdrawal = block_on(
19900 repositories
19901 .assets
19902 .get_dataset_version_withdrawal(only.version_id),
19903 )
19904 .expect("withdrawal lookup should succeed")
19905 .expect("withdrawal should be queryable");
19906 assert_eq!(withdrawal.reason, "contains invalid rows");
19907 }
19908
19909 #[test]
19910 fn repeated_dataset_version_withdrawal_is_idempotent_and_preserves_tombstone() {
19911 let repositories = AssetWorkflowRepositories::new();
19912 let service = repositories.service();
19913 let study = test_study("issue05_repeated_withdraw_study");
19914 repositories.seed_study(study.clone());
19915 repositories.grant_study_custodianship(study.study_id, "first-curator");
19916 repositories.grant_study_custodianship(study.study_id, "second-curator");
19917 let asset = test_asset(study.study_id, "issue05_repeated_withdraw_dataset");
19918 let version =
19919 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "faulty");
19920
19921 let first = block_on(service.withdraw_dataset_version(
19922 None,
19923 WithdrawDatasetVersionRequest {
19924 study_id: study.study_id,
19925 dataset_name: asset.name.clone(),
19926 version: "1.0.0".to_string(),
19927 reason: "first reason".to_string(),
19928 actor: Some("first-curator".to_string()),
19929 force: true,
19930 drop_lake_table: false,
19931 },
19932 ))
19933 .expect("first withdrawal should succeed");
19934 let second = block_on(service.withdraw_dataset_version(
19935 None,
19936 WithdrawDatasetVersionRequest {
19937 study_id: study.study_id,
19938 dataset_name: asset.name.clone(),
19939 version: "1.0.0".to_string(),
19940 reason: "second reason".to_string(),
19941 actor: Some("second-curator".to_string()),
19942 force: true,
19943 drop_lake_table: false,
19944 },
19945 ))
19946 .expect("second withdrawal should succeed");
19947
19948 assert!(!first.already_withdrawn);
19949 assert!(second.already_withdrawn);
19950 assert_eq!(second.withdrawal.reason, "first reason");
19951 assert_eq!(
19952 second.withdrawal.withdrawn_by.as_deref(),
19953 Some("first-curator")
19954 );
19955 let withdrawals = block_on(
19956 repositories
19957 .assets
19958 .list_dataset_version_withdrawals(asset.asset_id),
19959 )
19960 .expect("withdrawals should list");
19961 assert_eq!(withdrawals.len(), 1);
19962 assert_eq!(withdrawals[0].dataset_id, version.version_id);
19963 }
19964
19965 #[test]
19966 fn withdraw_dataset_version_requires_explicit_version_reason_and_force() {
19967 let repositories = AssetWorkflowRepositories::new();
19968 let service = repositories.service();
19969 let study = test_study("issue05_withdraw_validation_study");
19970 repositories.seed_study(study.clone());
19971 let asset = test_asset(study.study_id, "issue05_withdraw_validation_dataset");
19972 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "baseline");
19973
19974 for (version, reason, force, expected) in [
19975 ("latest", "bad", true, "explicit version"),
19976 ("1.0.0", " ", true, "reason"),
19977 ("1.0.0", "bad", false, "--force"),
19978 ] {
19979 let error = block_on(service.withdraw_dataset_version(
19980 None,
19981 WithdrawDatasetVersionRequest {
19982 study_id: study.study_id,
19983 dataset_name: asset.name.clone(),
19984 version: version.to_string(),
19985 reason: reason.to_string(),
19986 actor: None,
19987 force,
19988 drop_lake_table: false,
19989 },
19990 ))
19991 .expect_err("invalid withdrawal should fail");
19992 assert!(error.to_string().contains(expected), "{error}");
19993 }
19994 }
19995
19996 #[test]
19997 fn withdraw_dataset_version_requires_study_custodianship() {
19998 let repositories = AssetWorkflowRepositories::new();
19999 let service = repositories.service();
20000 let study = test_study("issue05_withdraw_authorization_study");
20001 repositories.seed_study(study.clone());
20002 repositories.grant_study_access(study.study_id, "reader@example.test");
20003 let asset = test_asset(study.study_id, "issue05_withdraw_authorization_dataset");
20004 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "baseline");
20005
20006 let error = block_on(service.withdraw_dataset_version(
20007 None,
20008 WithdrawDatasetVersionRequest {
20009 study_id: study.study_id,
20010 dataset_name: asset.name.clone(),
20011 version: "1.0.0".to_string(),
20012 reason: "bad rows".to_string(),
20013 actor: Some("reader@example.test".to_string()),
20014 force: true,
20015 drop_lake_table: false,
20016 },
20017 ))
20018 .expect_err("study access without custodianship should not authorize withdrawal");
20019
20020 assert!(
20021 error.to_string().contains("custodianship"),
20022 "unexpected error: {error}"
20023 );
20024 }
20025
20026 #[test]
20027 fn withdraw_dataset_version_removes_dataset_variable_links() {
20028 let repositories = AssetWorkflowRepositories::new();
20029 let service = repositories.service();
20030 let study = test_study("issue05_withdraw_links_study");
20031 let domain = test_domain(95_001, "issue05_withdraw_links_domain");
20032 repositories.seed_study(study.clone());
20033 repositories.grant_study_custodianship(study.study_id, "curator@example.test");
20034 repositories.seed_domain(domain.clone());
20035 let asset = test_asset(study.study_id, "issue05_withdraw_links_dataset");
20036 let version =
20037 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "linked");
20038 let variable = VariableRecord {
20039 variable_id: VariableId(95_001),
20040 domain_id: domain.domain_id,
20041 name: nc_name("record_id"),
20042 value_type_id: ValueTypeId(1),
20043 value_format: None,
20044 vocabulary_id: None,
20045 key_role: KeyRole::Record,
20046 description: None,
20047 note: None,
20048 ontology_namespace: None,
20049 ontology_class: None,
20050 };
20051 repositories.seed_variable(variable.clone());
20052 block_on(repositories.variables.link_dataset_variable(
20053 version.version_id,
20054 variable.variable_id,
20055 KeyRole::Record,
20056 ))
20057 .expect("dataset variable should link");
20058
20059 block_on(service.withdraw_dataset_version(
20060 None,
20061 WithdrawDatasetVersionRequest {
20062 study_id: study.study_id,
20063 dataset_name: asset.name.clone(),
20064 version: "1.0.0".to_string(),
20065 reason: "link cleanup test".to_string(),
20066 actor: Some("curator@example.test".to_string()),
20067 force: true,
20068 drop_lake_table: false,
20069 },
20070 ))
20071 .expect("withdrawal should succeed");
20072
20073 let links = block_on(
20074 repositories
20075 .variables
20076 .list_dataset_variables(version.version_id),
20077 )
20078 .expect("dataset links should list");
20079 assert!(links.is_empty());
20080 }
20081
20082 #[test]
20083 fn withdraw_dataset_version_drop_policy_is_safe_when_lake_table_is_missing() {
20084 let repositories = AssetWorkflowRepositories::new();
20085 let service = repositories.service();
20086 let study = test_study("issue05_missing_lake_table_study");
20087 repositories.seed_study(study.clone());
20088 repositories.grant_study_custodianship(study.study_id, "curator@example.test");
20089 let asset = test_asset(study.study_id, "issue05_missing_lake_table_dataset");
20090 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "missing table");
20091 let lake_root = temp_lake_root_path();
20092 let mut session = test_lake_session(&lake_root);
20093
20094 let result = block_on(service.withdraw_dataset_version(
20095 Some(&mut session),
20096 WithdrawDatasetVersionRequest {
20097 study_id: study.study_id,
20098 dataset_name: asset.name.clone(),
20099 version: "1.0.0".to_string(),
20100 reason: "drop policy test".to_string(),
20101 actor: Some("curator@example.test".to_string()),
20102 force: true,
20103 drop_lake_table: true,
20104 },
20105 ))
20106 .expect("withdrawal with missing lake table should succeed");
20107
20108 assert_eq!(
20109 result.lake_table_drop,
20110 LakeTableDropDisposition::DroppedIfExisted
20111 );
20112 assert!(result.withdrawal.drop_lake_table);
20113 let _ = fs::remove_dir_all(&lake_root);
20114 }
20115
20116 #[test]
20117 fn delete_dataset_asset_version_requires_accepted_concrete_plan_and_tombstones_version() {
20118 let repositories = AssetWorkflowRepositories::new();
20119 let service = repositories.service();
20120 let study = test_study("issue04_delete_dataset_version_study");
20121 repositories.seed_study(study.clone());
20122 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20123 let asset = test_asset(study.study_id, "issue04_delete_dataset_version_dataset");
20124 let first =
20125 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, false, "baseline");
20126 let second =
20127 seed_completed_dataset_version(&repositories, &asset, 2, 0, 0, true, "bad refresh");
20128 let lake_root = temp_lake_root_path();
20129 let mut session = test_lake_session(&lake_root);
20130
20131 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20132 target: DeleteTarget::AssetVersion(second.version_id),
20133 archive_mode: ArchiveMode::AssetArchive,
20134 cascade: false,
20135 force: false,
20136 reason: "bad refresh".to_string(),
20137 actor: Some("curator@example.org".to_string()),
20138 version_target: Some(ArchiveDeleteVersionTarget::Version {
20139 version_id: second.version_id,
20140 }),
20141 delete_all_versions: false,
20142 dry_run: false,
20143 }))
20144 .expect("planning should succeed")
20145 .plan
20146 .expect("accepted plan should be present");
20147
20148 let deleted =
20149 block_on(service.delete_dataset_asset_version(
20150 &mut session,
20151 DeleteDatasetAssetVersionRequest { plan },
20152 ))
20153 .expect("dataset version archive/delete should succeed");
20154
20155 assert_eq!(deleted.target_version.version_id, second.version_id);
20156 assert_eq!(deleted.tombstone.dataset_id, second.version_id);
20157 assert_eq!(deleted.tombstone.tombstone_state, "withdrawn");
20158 assert_eq!(
20159 deleted.lake_cleanup,
20160 LakeTableDropDisposition::DroppedIfExisted
20161 );
20162 assert!(deleted.tombstone.drop_lake_table);
20163 assert_eq!(deleted.retry_state, ArchiveDeleteRetryState::Completed);
20164 assert_eq!(
20165 deleted
20166 .promoted_latest_version
20167 .as_ref()
20168 .map(|version| version.version_id),
20169 Some(first.version_id)
20170 );
20171 assert_eq!(
20172 deleted
20173 .deletion
20174 .catalog
20175 .latest_version
20176 .as_ref()
20177 .map(|version| version.version_id),
20178 Some(first.version_id)
20179 );
20180 }
20181
20182 #[test]
20183 fn dataset_delete_planning_requires_custodianship_even_for_public_study() {
20184 let repositories = AssetWorkflowRepositories::new();
20185 let service = repositories.service();
20186 let study = test_study("issue04_public_dataset_delete_authorization_study");
20187 repositories.seed_study(study.clone());
20188 let asset = test_asset(
20189 study.study_id,
20190 "issue04_public_dataset_delete_authorization_dataset",
20191 );
20192 let version =
20193 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "baseline");
20194
20195 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20196 target: DeleteTarget::AssetVersion(version.version_id),
20197 archive_mode: ArchiveMode::AssetArchive,
20198 cascade: false,
20199 force: false,
20200 reason: "bad refresh".to_string(),
20201 actor: Some("reader@example.org".to_string()),
20202 version_target: Some(ArchiveDeleteVersionTarget::Version {
20203 version_id: version.version_id,
20204 }),
20205 delete_all_versions: false,
20206 dry_run: true,
20207 }))
20208 .expect("planning should return validation errors instead of failing");
20209
20210 assert!(!plan.accepted);
20211 let message = plan
20212 .validation_errors
20213 .iter()
20214 .find(|error| error.code == "unauthorized_actor")
20215 .map(|error| error.message.as_str())
20216 .expect("unauthorized actor error should be present");
20217 assert!(message.contains("custodianship"), "{message}");
20218 }
20219
20220 #[test]
20221 fn delete_dataset_asset_version_rejects_unresolved_execution_plan() {
20222 let repositories = AssetWorkflowRepositories::new();
20223 let service = repositories.service();
20224 let study = test_study("issue04_delete_dataset_version_unresolved_study");
20225 repositories.seed_study(study.clone());
20226 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20227 let asset = test_asset(
20228 study.study_id,
20229 "issue04_delete_dataset_version_unresolved_dataset",
20230 );
20231 let version =
20232 seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, true, "baseline");
20233 let lake_root = temp_lake_root_path();
20234 let mut session = test_lake_session(&lake_root);
20235 let mut plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20236 target: DeleteTarget::AssetVersion(version.version_id),
20237 archive_mode: ArchiveMode::AssetArchive,
20238 cascade: false,
20239 force: false,
20240 reason: "bad refresh".to_string(),
20241 actor: Some("curator@example.org".to_string()),
20242 version_target: Some(ArchiveDeleteVersionTarget::Version {
20243 version_id: version.version_id,
20244 }),
20245 delete_all_versions: false,
20246 dry_run: false,
20247 }))
20248 .expect("planning should succeed")
20249 .plan
20250 .expect("accepted plan should be present");
20251 plan.resolved_version = None;
20252
20253 let error =
20254 block_on(service.delete_dataset_asset_version(
20255 &mut session,
20256 DeleteDatasetAssetVersionRequest { plan },
20257 ))
20258 .expect_err("unresolved execution plan should fail");
20259
20260 assert!(error.to_string().contains("concrete resolved version"));
20261 assert!(
20262 block_on(
20263 repositories
20264 .assets
20265 .get_dataset_version_withdrawal(version.version_id)
20266 )
20267 .expect("withdrawal lookup should succeed")
20268 .is_none()
20269 );
20270 }
20271
20272 #[test]
20273 fn semantic_delete_dictionary_id_selectors_match_name_selectors_and_execute() {
20274 let repositories = AssetWorkflowRepositories::new();
20275 let service = AppService::new(AppRepositories {
20276 registrations: repositories.domains.clone(),
20277 domains: repositories.domains.clone(),
20278 studies: repositories.studies.clone(),
20279 study_domains: repositories.study_domains.clone(),
20280 study_governance: Arc::new(StatefulAssetStudyGovernanceRepository {
20281 state: repositories.state.clone(),
20282 }),
20283 assets: repositories.assets.clone(),
20284 variables: repositories.variables.clone(),
20285 vocabularies: repositories.vocabularies.clone(),
20286 entities: Arc::new(EmptyEntityRepository),
20287 transformations: repositories.transformations.clone(),
20288 tags: Arc::new(StatefulTagRepository {
20289 state: repositories.state.clone(),
20290 }),
20291 });
20292 let empty_domain = test_domain(97_001, "p1_20i_semantic_delete_empty_domain");
20293 let dictionary_domain = test_domain(97_002, "p1_20i_semantic_delete_dictionary_domain");
20294 repositories.seed_domain(empty_domain.clone());
20295 repositories.seed_domain(dictionary_domain.clone());
20296
20297 let domain_by_name = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20298 target: SemanticDeleteTarget::Domain {
20299 domain: DomainSelector::Name {
20300 name: empty_domain.name.as_str().to_string(),
20301 },
20302 },
20303 reason: "remove empty test domain".to_string(),
20304 actor: Some("curator@example.org".to_string()),
20305 dry_run: true,
20306 }))
20307 .expect("domain name delete plan should succeed")
20308 .plan
20309 .expect("domain name plan should be accepted");
20310 let domain_by_id = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20311 target: SemanticDeleteTarget::Domain {
20312 domain: DomainSelector::Id {
20313 domain_id: empty_domain.domain_id,
20314 },
20315 },
20316 reason: "remove empty test domain".to_string(),
20317 actor: Some("curator@example.org".to_string()),
20318 dry_run: true,
20319 }))
20320 .expect("domain id delete plan should succeed")
20321 .plan
20322 .expect("domain id plan should be accepted");
20323 assert_eq!(domain_by_id.target_identity, domain_by_name.target_identity);
20324 assert_eq!(
20325 domain_by_id.dependency_summary,
20326 domain_by_name.dependency_summary
20327 );
20328 let domain_deleted = block_on(service.execute_semantic_delete(
20329 ExecuteSemanticDeleteRequest {
20330 plan: domain_by_id,
20331 confirmed: true,
20332 },
20333 ))
20334 .expect("domain id delete should execute");
20335 assert!(domain_deleted.deleted);
20336
20337 let variable = test_variable(
20338 97_101,
20339 dictionary_domain.domain_id.0,
20340 "delete_me_variable",
20341 KeyRole::None,
20342 None,
20343 );
20344 repositories.seed_variable(variable.clone());
20345 let variable_by_name = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20346 target: SemanticDeleteTarget::Variable {
20347 variable: VariableSelector::Name {
20348 domain: DomainSelector::Name {
20349 name: dictionary_domain.name.as_str().to_string(),
20350 },
20351 name: variable.name.as_str().to_string(),
20352 },
20353 },
20354 reason: "remove test variable".to_string(),
20355 actor: Some("curator@example.org".to_string()),
20356 dry_run: true,
20357 }))
20358 .expect("variable name delete plan should succeed")
20359 .plan
20360 .expect("variable name plan should be accepted");
20361 let variable_by_id = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20362 target: SemanticDeleteTarget::Variable {
20363 variable: VariableSelector::Id {
20364 variable_id: variable.variable_id,
20365 },
20366 },
20367 reason: "remove test variable".to_string(),
20368 actor: Some("curator@example.org".to_string()),
20369 dry_run: true,
20370 }))
20371 .expect("variable id delete plan should succeed")
20372 .plan
20373 .expect("variable id plan should be accepted");
20374 assert_eq!(
20375 variable_by_id.target_identity,
20376 variable_by_name.target_identity
20377 );
20378 assert_eq!(
20379 variable_by_id.dependency_summary,
20380 variable_by_name.dependency_summary
20381 );
20382 let variable_deleted = block_on(service.execute_semantic_delete(
20383 ExecuteSemanticDeleteRequest {
20384 plan: variable_by_id,
20385 confirmed: true,
20386 },
20387 ))
20388 .expect("variable id delete should execute");
20389 assert!(variable_deleted.deleted);
20390
20391 let vocabulary = test_vocabulary(
20392 97_201,
20393 dictionary_domain.domain_id.0,
20394 "delete_me_vocabulary",
20395 );
20396 repositories.seed_vocabulary(vocabulary.clone());
20397 let vocabulary_by_name =
20398 block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20399 target: SemanticDeleteTarget::Vocabulary {
20400 vocabulary: VocabularySelector::Name {
20401 domain: DomainSelector::Name {
20402 name: dictionary_domain.name.as_str().to_string(),
20403 },
20404 name: vocabulary.name.as_str().to_string(),
20405 },
20406 },
20407 reason: "remove test vocabulary".to_string(),
20408 actor: Some("curator@example.org".to_string()),
20409 dry_run: true,
20410 }))
20411 .expect("vocabulary name delete plan should succeed")
20412 .plan
20413 .expect("vocabulary name plan should be accepted");
20414 let vocabulary_by_id = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20415 target: SemanticDeleteTarget::Vocabulary {
20416 vocabulary: VocabularySelector::Id {
20417 vocabulary_id: vocabulary.vocabulary_id,
20418 },
20419 },
20420 reason: "remove test vocabulary".to_string(),
20421 actor: Some("curator@example.org".to_string()),
20422 dry_run: true,
20423 }))
20424 .expect("vocabulary id delete plan should succeed")
20425 .plan
20426 .expect("vocabulary id plan should be accepted");
20427 assert_eq!(
20428 vocabulary_by_id.target_identity,
20429 vocabulary_by_name.target_identity
20430 );
20431 assert_eq!(
20432 vocabulary_by_id.dependency_summary,
20433 vocabulary_by_name.dependency_summary
20434 );
20435 let vocabulary_deleted = block_on(service.execute_semantic_delete(
20436 ExecuteSemanticDeleteRequest {
20437 plan: vocabulary_by_id,
20438 confirmed: true,
20439 },
20440 ))
20441 .expect("vocabulary id delete should execute");
20442 assert!(vocabulary_deleted.deleted);
20443
20444 let missing = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20445 target: SemanticDeleteTarget::Variable {
20446 variable: VariableSelector::Id {
20447 variable_id: VariableId(979_999),
20448 },
20449 },
20450 reason: "remove missing variable".to_string(),
20451 actor: Some("curator@example.org".to_string()),
20452 dry_run: true,
20453 }))
20454 .expect("missing id delete plan should reject without infrastructure failure");
20455 assert!(!missing.accepted);
20456 assert!(
20457 missing
20458 .validation_errors
20459 .iter()
20460 .any(|error| error.code == "target_not_found")
20461 );
20462 }
20463
20464 #[test]
20465 fn delete_datafile_asset_version_removes_managed_file_and_active_metadata() {
20466 let repositories = AssetWorkflowRepositories::new();
20467 let service = repositories.service();
20468 let study = test_study("issue05_delete_datafile_version_study");
20469 repositories.seed_study(study.clone());
20470 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20471 let lake_root = temp_lake_root_path();
20472 let mut session = test_lake_session(&lake_root);
20473 let mut asset = test_asset(study.study_id, "issue05_delete_datafile_version_file");
20474 asset.asset_type = AssetType::File;
20475 let first = seed_completed_datafile_version(
20476 &repositories,
20477 &lake_root,
20478 &asset,
20479 (1, 0, 0),
20480 false,
20481 b"first",
20482 );
20483 let second = seed_completed_datafile_version(
20484 &repositories,
20485 &lake_root,
20486 &asset,
20487 (2, 0, 0),
20488 true,
20489 b"second",
20490 );
20491 let datafile = block_on(repositories.assets.get_datafile(second.version_id))
20492 .expect("datafile lookup should succeed")
20493 .expect("datafile should exist");
20494 let stored_path =
20495 datafile_storage_uri_path(&lake_root.display().to_string(), &datafile.storage_uri)
20496 .expect("managed datafile path should resolve");
20497 assert!(stored_path.exists());
20498 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20499 target: DeleteTarget::AssetVersion(second.version_id),
20500 archive_mode: ArchiveMode::AssetArchive,
20501 cascade: false,
20502 force: false,
20503 reason: "delete obsolete file version".to_string(),
20504 actor: Some("curator@example.org".to_string()),
20505 version_target: Some(ArchiveDeleteVersionTarget::Version {
20506 version_id: second.version_id,
20507 }),
20508 delete_all_versions: false,
20509 dry_run: false,
20510 }))
20511 .expect("planning should succeed")
20512 .plan
20513 .expect("accepted plan should be present");
20514
20515 let deleted = block_on(service.delete_datafile_asset_version(
20516 &mut session,
20517 DeleteDataFileAssetVersionRequest { plan },
20518 ))
20519 .expect("datafile version delete should succeed");
20520
20521 assert_eq!(deleted.target_version.version_id, second.version_id);
20522 assert_eq!(
20523 deleted.managed_file_cleanup,
20524 ArchiveDeleteCleanupDisposition::Completed
20525 );
20526 assert_eq!(
20527 deleted
20528 .promoted_latest_version
20529 .as_ref()
20530 .map(|version| version.version_id),
20531 Some(first.version_id)
20532 );
20533 assert!(!stored_path.exists());
20534 assert!(
20535 block_on(repositories.assets.get_datafile(second.version_id))
20536 .expect("datafile lookup should succeed")
20537 .is_none()
20538 );
20539 assert!(
20540 block_on(repositories.assets.get_asset_version(second.version_id))
20541 .expect("asset version lookup should succeed")
20542 .is_none()
20543 );
20544 }
20545
20546 #[test]
20547 fn delete_datafile_asset_version_skips_missing_managed_file_but_removes_metadata() {
20548 let repositories = AssetWorkflowRepositories::new();
20549 let service = repositories.service();
20550 let study = test_study("issue07_missing_datafile_cleanup_study");
20551 repositories.seed_study(study.clone());
20552 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20553 let lake_root = temp_lake_root_path();
20554 let mut session = test_lake_session(&lake_root);
20555 let mut asset = test_asset(study.study_id, "issue07_missing_datafile_cleanup_file");
20556 asset.asset_type = AssetType::File;
20557 let version = seed_completed_datafile_version(
20558 &repositories,
20559 &lake_root,
20560 &asset,
20561 (1, 0, 0),
20562 true,
20563 b"already archived",
20564 );
20565 let datafile = block_on(repositories.assets.get_datafile(version.version_id))
20566 .expect("datafile lookup should succeed")
20567 .expect("datafile should exist");
20568 let stored_path =
20569 datafile_storage_uri_path(&lake_root.display().to_string(), &datafile.storage_uri)
20570 .expect("managed datafile path should resolve");
20571 fs::remove_file(&stored_path).expect("fixture should remove staged file");
20572 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20573 target: DeleteTarget::AssetVersion(version.version_id),
20574 archive_mode: ArchiveMode::AssetArchive,
20575 cascade: false,
20576 force: false,
20577 reason: "delete missing payload record".to_string(),
20578 actor: Some("curator@example.org".to_string()),
20579 version_target: Some(ArchiveDeleteVersionTarget::Version {
20580 version_id: version.version_id,
20581 }),
20582 delete_all_versions: false,
20583 dry_run: false,
20584 }))
20585 .expect("planning should succeed")
20586 .plan
20587 .expect("accepted plan should be present");
20588
20589 let deleted = block_on(service.delete_datafile_asset_version(
20590 &mut session,
20591 DeleteDataFileAssetVersionRequest { plan },
20592 ))
20593 .expect("datafile version delete should succeed when payload is missing");
20594
20595 assert_eq!(
20596 deleted.managed_file_cleanup,
20597 ArchiveDeleteCleanupDisposition::Skipped
20598 );
20599 assert_eq!(
20600 deleted.metadata_cleanup,
20601 ArchiveDeleteCleanupDisposition::Completed
20602 );
20603 assert_eq!(deleted.retry_state, ArchiveDeleteRetryState::Completed);
20604 assert!(
20605 block_on(repositories.assets.get_asset_version(version.version_id))
20606 .expect("asset version lookup should succeed")
20607 .is_none()
20608 );
20609 }
20610
20611 #[test]
20612 fn delete_dataset_asset_removes_variable_links_and_unblocks_variable_delete() {
20613 let repositories = AssetWorkflowRepositories::new();
20614 let service = repositories.service();
20615 let study = test_study("issue06_delete_dataset_asset_study");
20616 let domain = test_domain(96_001, "issue06_delete_dataset_asset_domain");
20617 repositories.seed_study(study.clone());
20618 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20619 repositories.seed_domain(domain.clone());
20620 let lake_root = temp_lake_root_path();
20621 let mut session = test_lake_session(&lake_root);
20622 let asset = test_asset(study.study_id, "issue06_delete_dataset_asset");
20623 let first = seed_completed_dataset_version(&repositories, &asset, 1, 0, 0, false, "first");
20624 let second = seed_completed_dataset_version(&repositories, &asset, 2, 0, 0, true, "second");
20625 let variable = test_variable(96_001, domain.domain_id.0, "Sex", KeyRole::None, None);
20626 repositories.seed_variable(variable.clone());
20627 block_on(repositories.variables.link_dataset_variable(
20628 first.version_id,
20629 variable.variable_id,
20630 KeyRole::None,
20631 ))
20632 .expect("first dataset variable should link");
20633 block_on(repositories.variables.link_dataset_variable(
20634 second.version_id,
20635 variable.variable_id,
20636 KeyRole::None,
20637 ))
20638 .expect("second dataset variable should link");
20639
20640 let blocked_plan = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20641 target: SemanticDeleteTarget::Variable {
20642 variable: VariableSelector::Name {
20643 domain: domain.name.as_str().into(),
20644 name: variable.name.as_str().to_string(),
20645 },
20646 },
20647 reason: "wrong type".to_string(),
20648 actor: Some("curator@example.org".to_string()),
20649 dry_run: true,
20650 }))
20651 .expect("semantic delete planning should succeed before dataset delete")
20652 .plan
20653 .expect("accepted semantic delete plan should be present before dataset delete");
20654 assert_eq!(blocked_plan.disposition, SemanticDeleteDisposition::Blocked);
20655 assert!(blocked_plan.dependency_summary.has_blockers());
20656 assert!(
20657 blocked_plan
20658 .dependency_summary
20659 .historical_blockers
20660 .iter()
20661 .any(|dependency| dependency
20662 .description
20663 .contains("dataset issue06_delete_dataset_asset v1.0.0")),
20664 "dataset-variable blockers should name the blocking dataset and version"
20665 );
20666 let id_blocked_plan = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20667 target: SemanticDeleteTarget::Variable {
20668 variable: VariableSelector::Id {
20669 variable_id: variable.variable_id,
20670 },
20671 },
20672 reason: "wrong type".to_string(),
20673 actor: Some("curator@example.org".to_string()),
20674 dry_run: true,
20675 }))
20676 .expect("semantic delete planning should accept variable id selector")
20677 .plan
20678 .expect("id-selected blocked semantic delete plan should be accepted");
20679 assert_eq!(
20680 id_blocked_plan.disposition,
20681 SemanticDeleteDisposition::Blocked
20682 );
20683 assert_eq!(
20684 id_blocked_plan.dependency_summary,
20685 blocked_plan.dependency_summary
20686 );
20687
20688 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20689 target: DeleteTarget::Asset(asset.asset_id),
20690 archive_mode: ArchiveMode::AssetArchive,
20691 cascade: true,
20692 force: false,
20693 reason: "delete obsolete dataset asset".to_string(),
20694 actor: Some("curator@example.org".to_string()),
20695 version_target: None,
20696 delete_all_versions: true,
20697 dry_run: false,
20698 }))
20699 .expect("dataset delete planning should succeed")
20700 .plan
20701 .expect("accepted dataset delete plan should be present");
20702
20703 let deleted = block_on(service.delete_asset(&mut session, DeleteAssetRequest { plan }))
20704 .expect("whole dataset delete should succeed");
20705
20706 assert_eq!(deleted.deleted_versions, 2);
20707 assert_eq!(deleted.failed_versions, 0);
20708 assert!(
20709 block_on(
20710 repositories
20711 .variables
20712 .list_dataset_variables(first.version_id)
20713 )
20714 .expect("first dataset links should list")
20715 .is_empty()
20716 );
20717 assert!(
20718 block_on(
20719 repositories
20720 .variables
20721 .list_dataset_variables(second.version_id)
20722 )
20723 .expect("second dataset links should list")
20724 .is_empty()
20725 );
20726 assert!(
20727 block_on(
20728 repositories
20729 .variables
20730 .list_dataset_variables_for_variable(variable.variable_id)
20731 )
20732 .expect("variable dataset links should list")
20733 .is_empty()
20734 );
20735
20736 let unblocked_plan = block_on(service.plan_semantic_delete(PlanSemanticDeleteRequest {
20737 target: SemanticDeleteTarget::Variable {
20738 variable: VariableSelector::Name {
20739 domain: domain.name.as_str().into(),
20740 name: variable.name.as_str().to_string(),
20741 },
20742 },
20743 reason: "wrong type".to_string(),
20744 actor: Some("curator@example.org".to_string()),
20745 dry_run: true,
20746 }))
20747 .expect("semantic delete planning should succeed after dataset delete")
20748 .plan
20749 .expect("accepted semantic delete plan should be present after dataset delete");
20750 assert_eq!(
20751 unblocked_plan.disposition,
20752 SemanticDeleteDisposition::PhysicalDeleteAllowed
20753 );
20754 assert!(!unblocked_plan.dependency_summary.has_blockers());
20755 }
20756
20757 #[test]
20758 fn delete_asset_processes_all_datafile_versions_and_removes_asset() {
20759 let repositories = AssetWorkflowRepositories::new();
20760 let service = repositories.service();
20761 let study = test_study("issue06_delete_file_asset_study");
20762 repositories.seed_study(study.clone());
20763 repositories.grant_study_custodianship(study.study_id, "curator@example.org");
20764 let lake_root = temp_lake_root_path();
20765 let mut session = test_lake_session(&lake_root);
20766 let mut asset = test_asset(study.study_id, "issue06_delete_file_asset");
20767 asset.asset_type = AssetType::File;
20768 seed_completed_datafile_version(&repositories, &lake_root, &asset, (1, 0, 0), false, b"a");
20769 seed_completed_datafile_version(&repositories, &lake_root, &asset, (2, 0, 0), true, b"b");
20770 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20771 target: DeleteTarget::Asset(asset.asset_id),
20772 archive_mode: ArchiveMode::AssetArchive,
20773 cascade: true,
20774 force: false,
20775 reason: "delete obsolete file asset".to_string(),
20776 actor: Some("curator@example.org".to_string()),
20777 version_target: None,
20778 delete_all_versions: true,
20779 dry_run: false,
20780 }))
20781 .expect("planning should succeed")
20782 .plan
20783 .expect("accepted plan should be present");
20784
20785 let deleted = block_on(service.delete_asset(&mut session, DeleteAssetRequest { plan }))
20786 .expect("whole asset delete should succeed");
20787
20788 assert_eq!(deleted.deleted_versions, 2);
20789 assert_eq!(deleted.failed_versions, 0);
20790 assert_eq!(
20791 deleted.metadata_cleanup,
20792 ArchiveDeleteCleanupDisposition::Completed
20793 );
20794 assert!(
20795 block_on(repositories.assets.get_asset_by_id(asset.asset_id))
20796 .expect("asset lookup should succeed")
20797 .is_none()
20798 );
20799 assert!(
20800 block_on(repositories.assets.list_asset_versions(asset.asset_id))
20801 .expect("version list should succeed")
20802 .is_empty()
20803 );
20804 }
20805
20806 #[test]
20807 fn delete_archived_study_cascades_after_archive_ingest_and_keeps_archive_study() {
20808 let repositories = AssetWorkflowRepositories::new();
20809 let service = repositories.service();
20810 let source_study = test_study("issue03_delete_archived_source_study");
20811 repositories.seed_study(source_study.clone());
20812 repositories
20813 .state
20814 .custodians
20815 .lock()
20816 .expect("study custodian lock should not be poisoned")
20817 .push(StudyCustodianLinkRecord {
20818 study_id: source_study.study_id,
20819 user_id: "curator@example.org".to_string(),
20820 is_primary: true,
20821 date_granted: None,
20822 granted_by: None,
20823 });
20824 let lake_root = temp_lake_root_path();
20825 let mut session = test_lake_session(&lake_root);
20826 let mut asset = test_asset(source_study.study_id, "issue03_delete_source_file");
20827 asset.asset_type = AssetType::File;
20828 seed_completed_datafile_version(
20829 &repositories,
20830 &lake_root,
20831 &asset,
20832 (1, 0, 0),
20833 true,
20834 b"source",
20835 );
20836 let archived = block_on(service.archive_study_for_deletion(
20837 &mut session,
20838 ArchiveStudyForDeletionRequest {
20839 study_id: source_study.study_id,
20840 reason: "retire source study".to_string(),
20841 actor: Some("curator@example.org".to_string()),
20842 overwrite_existing_exports: true,
20843 },
20844 ))
20845 .expect("study archive should be ingested before delete");
20846 let archive_study_id = archived.prepared.archive_study.study_id;
20847 let plan = block_on(service.plan_archive_delete(PlanArchiveDeleteRequest {
20848 target: DeleteTarget::Study(source_study.study_id),
20849 archive_mode: ArchiveMode::StudyArchive,
20850 cascade: true,
20851 force: false,
20852 reason: "retire source study".to_string(),
20853 actor: Some("curator@example.org".to_string()),
20854 version_target: None,
20855 delete_all_versions: false,
20856 dry_run: false,
20857 }))
20858 .expect("study delete planning should succeed")
20859 .plan
20860 .expect("accepted plan should be present");
20861
20862 let deleted = block_on(service.delete_archived_study(
20863 &mut session,
20864 DeleteArchivedStudyRequest {
20865 plan,
20866 archive: archived,
20867 },
20868 ))
20869 .expect("archived study delete should succeed");
20870
20871 assert_eq!(deleted.deleted_assets, 1);
20872 assert_eq!(deleted.deleted_versions, 1);
20873 assert_eq!(
20874 deleted.metadata_cleanup,
20875 ArchiveDeleteCleanupDisposition::Completed
20876 );
20877 assert!(
20878 block_on(repositories.studies.get_study_by_id(source_study.study_id))
20879 .expect("source study lookup should succeed")
20880 .is_none()
20881 );
20882 assert!(
20883 block_on(repositories.studies.get_study_by_id(archive_study_id))
20884 .expect("archive study lookup should succeed")
20885 .is_some()
20886 );
20887 assert!(
20888 block_on(repositories.assets.get_asset_by_id(asset.asset_id))
20889 .expect("source asset lookup should succeed")
20890 .is_none()
20891 );
20892 }
20893
20894 #[test]
20895 fn metadata_workflows_extract_inline_tags_and_clean_text() {
20896 let repositories = AssetWorkflowRepositories::new();
20897 let service = repositories.service();
20898 let study = test_study("p2_5_tagged_asset_study");
20899 repositories.seed_study(study.clone());
20900 let domain = DomainRecord {
20901 description: Some("Clinical #Core domain".to_string()),
20902 ..test_domain(90_501, "p2_5_tagged_domain")
20903 };
20904
20905 let registered_domain = block_on(service.register_domain(RegisterDomainRequest {
20906 domain: domain.clone(),
20907 }))
20908 .expect("domain with inline tag should register");
20909 assert_eq!(
20910 registered_domain.description.as_deref(),
20911 Some("Clinical domain")
20912 );
20913 assert_eq!(
20914 block_on(service.get_tags(GetTagsRequest {
20915 target: TagAttachmentTarget::Domain(domain.domain_id)
20916 }))
20917 .expect("domain tags should read")
20918 .tags
20919 .into_iter()
20920 .map(|tag| tag.name)
20921 .collect::<Vec<_>>(),
20922 vec!["core".to_string()]
20923 );
20924
20925 block_on(service.set_tags(SetTagsRequest {
20926 target: TagAttachmentTarget::Asset(AssetId(uuid::Uuid::from_u128(90_502))),
20927 tags: vec!["Curated".to_string()],
20928 }))
20929 .expect("explicit asset tags should set");
20930 let asset = AssetRecord {
20931 asset_id: AssetId(uuid::Uuid::from_u128(90_502)),
20932 description: Some("Baseline #FollowUp extract".to_string()),
20933 ..test_asset(study.study_id, "p2_5_tagged_asset")
20934 };
20935 let catalog = block_on(service.create_asset(CreateAssetRequest {
20936 asset: asset.clone(),
20937 }))
20938 .expect("asset with inline tag should create");
20939 assert_eq!(
20940 catalog.asset.description.as_deref(),
20941 Some("Baseline extract")
20942 );
20943 assert_eq!(
20944 block_on(service.get_tags(GetTagsRequest {
20945 target: TagAttachmentTarget::Asset(asset.asset_id)
20946 }))
20947 .expect("asset tags should read")
20948 .tags
20949 .into_iter()
20950 .map(|tag| tag.name)
20951 .collect::<Vec<_>>(),
20952 vec!["curated".to_string(), "followup".to_string()]
20953 );
20954 }
20955
20956 #[test]
20957 fn asset_version_and_variable_workflows_extract_inline_tags() {
20958 let repositories = AssetWorkflowRepositories::new();
20959 let service = repositories.service();
20960 let study = test_study("p2_5_tagged_version_study");
20961 repositories.seed_study(study.clone());
20962 let domain = test_domain(90_503, "p2_5_variable_domain");
20963 repositories.seed_domain(domain.clone());
20964 let asset = test_asset(study.study_id, "p2_5_versioned_asset");
20965 block_on(service.create_asset(CreateAssetRequest {
20966 asset: asset.clone(),
20967 }))
20968 .expect("asset creation should succeed");
20969
20970 let version = test_asset_version(asset.asset_id, 1, 0, 0, "First #Baseline version");
20971 let catalog = block_on(service.create_asset_version(CreateAssetVersionRequest {
20972 classification: ahri_tre_types::IngestClassification::derived_high(),
20973 version: version.clone(),
20974 }))
20975 .expect("asset version should create");
20976 let latest = catalog
20977 .latest_version
20978 .expect("latest version should be returned");
20979 assert_eq!(latest.version_note.as_deref(), Some("First version"));
20980 assert_eq!(
20981 block_on(service.get_tags(GetTagsRequest {
20982 target: TagAttachmentTarget::AssetVersion(version.version_id)
20983 }))
20984 .expect("version tags should read")
20985 .tags
20986 .into_iter()
20987 .map(|tag| tag.name)
20988 .collect::<Vec<_>>(),
20989 vec!["baseline".to_string()]
20990 );
20991
20992 let registered = block_on(service.register_variable(RegisterVariableRequest {
20993 domain_id: domain.domain_id,
20994 variable_id: None,
20995 name: nc_name("visit_status"),
20996 value_type_id: None,
20997 value_type: Some("xsd:string".to_string()),
20998 value_format: None,
20999 vocabulary_id: None,
21000 key_role: KeyRole::None,
21001 description: Some("Visit #Status".to_string()),
21002 note: None,
21003 ontology_namespace: None,
21004 ontology_class: None,
21005 force_metadata_updates: false,
21006 }))
21007 .expect("variable should register");
21008 assert_eq!(registered.variable.description.as_deref(), Some("Visit"));
21009 assert_eq!(
21010 block_on(service.get_tags(GetTagsRequest {
21011 target: TagAttachmentTarget::Variable(registered.variable.variable_id)
21012 }))
21013 .expect("variable tags should read")
21014 .tags
21015 .into_iter()
21016 .map(|tag| tag.name)
21017 .collect::<Vec<_>>(),
21018 vec!["status".to_string()]
21019 );
21020 }
21021
21022 #[test]
21023 fn explicit_tag_management_replaces_target_tags() {
21024 let service = AssetWorkflowRepositories::new().service();
21025 let target = TagAttachmentTarget::Asset(AssetId(uuid::Uuid::from_u128(90_504)));
21026
21027 block_on(service.set_tags(SetTagsRequest {
21028 target: target.clone(),
21029 tags: vec!["First".to_string(), "#Second Tag".to_string()],
21030 }))
21031 .expect("initial tags should set");
21032 let replaced = block_on(service.set_tags(SetTagsRequest {
21033 target: target.clone(),
21034 tags: vec!["Replacement".to_string()],
21035 }))
21036 .expect("replacement tags should set");
21037
21038 assert_eq!(
21039 replaced
21040 .tags
21041 .into_iter()
21042 .map(|tag| tag.name)
21043 .collect::<Vec<_>>(),
21044 vec!["replacement".to_string()]
21045 );
21046 assert_eq!(
21047 block_on(service.get_tags(GetTagsRequest { target }))
21048 .expect("target tags should read")
21049 .tags
21050 .into_iter()
21051 .map(|tag| tag.name)
21052 .collect::<Vec<_>>(),
21053 vec!["replacement".to_string()]
21054 );
21055 }
21056
21057 #[test]
21058 fn explicit_tag_management_accepts_semantic_target_ids() {
21059 let service = AssetWorkflowRepositories::new().service();
21060 let targets = [
21061 TagAttachmentTarget::Domain(DomainId(90_505)),
21062 TagAttachmentTarget::Study(StudyId(uuid::Uuid::from_u128(90_506))),
21063 TagAttachmentTarget::Variable(VariableId(90_507)),
21064 TagAttachmentTarget::Entity(EntityId(90_508)),
21065 TagAttachmentTarget::EntityRelation(EntityRelationId(90_509)),
21066 ];
21067
21068 for target in targets {
21069 block_on(service.set_tags(SetTagsRequest {
21070 target: target.clone(),
21071 tags: vec!["Semantic".to_string()],
21072 }))
21073 .expect("semantic target id tags should set");
21074
21075 assert_eq!(
21076 block_on(service.get_tags(GetTagsRequest {
21077 target: target.clone()
21078 }))
21079 .expect("semantic target id tags should read")
21080 .tags
21081 .into_iter()
21082 .map(|tag| tag.name)
21083 .collect::<Vec<_>>(),
21084 vec!["semantic".to_string()]
21085 );
21086 }
21087 }
21088
21089 #[test]
21090 fn create_asset_rejects_duplicate_name_in_study() {
21091 let repositories = AssetWorkflowRepositories::new();
21092 let service = repositories.service();
21093 let study = test_study("p1_11_duplicate_asset_study");
21094 repositories.seed_study(study.clone());
21095 let asset = test_asset(study.study_id, "p1_11_duplicate_asset");
21096
21097 block_on(service.create_asset(CreateAssetRequest {
21098 asset: asset.clone(),
21099 }))
21100 .expect("first asset creation should succeed");
21101
21102 let duplicate = AssetRecord {
21103 asset_id: AssetId(uuid::Uuid::new_v4()),
21104 ..asset
21105 };
21106 let error = block_on(service.create_asset(CreateAssetRequest { asset: duplicate }))
21107 .expect_err("duplicate asset name should fail");
21108
21109 match error {
21110 AppError::Validation(message) => assert!(message.contains("asset already exists")),
21111 other => panic!("expected validation error, got {other:?}"),
21112 }
21113 }
21114
21115 #[test]
21116 fn create_asset_version_marks_new_version_latest_and_exposes_metadata() {
21117 let repositories = AssetWorkflowRepositories::new();
21118 let service = repositories.service();
21119 let study = test_study("p1_11_version_study");
21120 repositories.seed_study(study.clone());
21121 let asset = test_asset(study.study_id, "p1_11_versioned_asset");
21122 block_on(service.create_asset(CreateAssetRequest {
21123 asset: asset.clone(),
21124 }))
21125 .expect("asset creation should succeed");
21126
21127 let first = test_asset_version(asset.asset_id, 1, 0, 0, "original import");
21128 let first_catalog = block_on(service.create_asset_version(CreateAssetVersionRequest {
21129 classification: ahri_tre_types::IngestClassification::derived_high(),
21130 version: first.clone(),
21131 }))
21132 .expect("first version creation should succeed");
21133 assert_eq!(
21134 first_catalog.latest_version.as_ref().map(|version| (
21135 version.version_id,
21136 version.version_label.as_deref(),
21137 version.is_latest
21138 )),
21139 Some((first.version_id, Some("v1.0.0"), Some(true)))
21140 );
21141
21142 let second = test_asset_version(asset.asset_id, 1, 1, 0, "curated refresh");
21143 let second_catalog = block_on(service.create_asset_version(CreateAssetVersionRequest {
21144 classification: ahri_tre_types::IngestClassification::derived_high(),
21145 version: second.clone(),
21146 }))
21147 .expect("second version creation should succeed");
21148
21149 assert_eq!(second_catalog.versions.len(), 2);
21150 assert_eq!(
21151 second_catalog
21152 .latest_version
21153 .as_ref()
21154 .map(|version| version.version_id),
21155 Some(second.version_id)
21156 );
21157 assert!(
21158 second_catalog
21159 .versions
21160 .iter()
21161 .any(|version| version.version_id == first.version_id
21162 && version.version_label.as_deref() == Some("v1.0.0")
21163 && version.is_latest == Some(false))
21164 );
21165 assert!(
21166 second_catalog
21167 .versions
21168 .iter()
21169 .any(|version| version.version_id == second.version_id
21170 && version.version_label.as_deref() == Some("v1.1.0")
21171 && version.is_latest == Some(true))
21172 );
21173
21174 let by_name = block_on(service.get_asset(GetAssetRequest {
21175 asset_id: None,
21176 study_id: Some(study.study_id),
21177 asset_name: Some(asset.name.clone()),
21178 }))
21179 .expect("asset should be retrievable by study and name");
21180 assert_eq!(by_name, second_catalog);
21181 }
21182
21183 #[test]
21184 fn create_asset_version_rejects_duplicate_coordinates_and_non_latest_requests() {
21185 let repositories = AssetWorkflowRepositories::new();
21186 let service = repositories.service();
21187 let study = test_study("p1_11_version_validation_study");
21188 repositories.seed_study(study.clone());
21189 let asset = test_asset(study.study_id, "p1_11_validated_asset");
21190 block_on(service.create_asset(CreateAssetRequest {
21191 asset: asset.clone(),
21192 }))
21193 .expect("asset creation should succeed");
21194
21195 let first = test_asset_version(asset.asset_id, 1, 0, 0, "original import");
21196 block_on(service.create_asset_version(CreateAssetVersionRequest {
21197 classification: ahri_tre_types::IngestClassification::derived_high(),
21198 version: first.clone(),
21199 }))
21200 .expect("first version creation should succeed");
21201
21202 let duplicate = AssetVersionRecord {
21203 version_id: VersionId(uuid::Uuid::new_v4()),
21204 ..first
21205 };
21206 let duplicate_error = block_on(service.create_asset_version(CreateAssetVersionRequest {
21207 version: duplicate,
21208 classification: ahri_tre_types::IngestClassification::derived_high(),
21209 }))
21210 .expect_err("duplicate version coordinates should fail");
21211 assert!(matches!(duplicate_error, AppError::Conflict(_)));
21212
21213 let non_latest = AssetVersionRecord {
21214 is_latest: Some(false),
21215 ..test_asset_version(asset.asset_id, 1, 1, 0, "non-latest request")
21216 };
21217 let non_latest_error = block_on(service.create_asset_version(CreateAssetVersionRequest {
21218 classification: ahri_tre_types::IngestClassification::derived_high(),
21219 version: non_latest,
21220 }))
21221 .expect_err("new versions must become latest");
21222 match non_latest_error {
21223 AppError::Validation(message) => assert!(message.contains("latest")),
21224 other => panic!("expected validation error, got {other:?}"),
21225 }
21226 }
21227
21228 #[test]
21229 fn datafile_list_workflow_resolves_study_scope_and_hides_missing_scope() {
21230 let repositories = AssetWorkflowRepositories::new();
21231 let service = repositories.service();
21232 let study = test_study("ticket_07_empty_study");
21233 repositories.seed_study(study.clone());
21234
21235 let listed = block_on(service.list_datafiles(crate::ListDataFilesRequest {
21236 study: StudySelector::Name {
21237 domain: None,
21238 name: study.name.as_str().to_string(),
21239 },
21240 include_versions: true,
21241 }))
21242 .expect("visible Study should return an empty Datafile catalogue");
21243 assert_eq!(listed.study_id, study.study_id);
21244 assert_eq!(listed.study_name, study.name);
21245 assert!(listed.datafiles.is_empty());
21246
21247 let error = block_on(service.list_datafiles(crate::ListDataFilesRequest {
21248 study: StudySelector::Id {
21249 study_id: StudyId(uuid::Uuid::new_v4()),
21250 },
21251 include_versions: true,
21252 }))
21253 .expect_err("missing Study scope should fail");
21254 assert!(matches!(error, AppError::NotFound(_)));
21255 }
21256
21257 #[test]
21258 fn ingest_file_streams_fixture_to_encrypted_file_asset_with_provenance() {
21259 let repositories = AssetWorkflowRepositories::new();
21260 let service = repositories.service();
21261 let study = test_study("p1_12_ingest_study");
21262 repositories.seed_study(study.clone());
21263 let source_path = workspace_file("data/ingest.csv");
21264 assert!(
21265 source_path.is_file(),
21266 "P1.12 ingest fixture should exist at {}",
21267 source_path.display()
21268 );
21269 let lake_root = temp_lake_root_path();
21270 let transformation = test_transformation(
21271 95,
21272 TransformationType::Ingest,
21273 "ingest governed source file",
21274 );
21275
21276 let ingested = block_on(service.ingest_file(IngestFileRequest {
21277 classification: ahri_tre_types::IngestClassification::derived_high(),
21278 study_id: study.study_id,
21279 asset_id: None,
21280 asset_name: nc_name("p1_12_ingested_file"),
21281 version_id: None,
21282 source: IngestFileSource::LocalPath {
21283 path: source_path.display().to_string(),
21284 },
21285 lake_root: lake_root.display().to_string(),
21286 edam_format: None,
21287 compress: true,
21288 encrypt: None,
21289 description: Some("P1.12 encrypted file asset".to_string()),
21290 agent_instructions: Some("Use as a governed raw file input".to_string()),
21291 version_note: Some("Original encrypted ingest".to_string()),
21292 transformation: transformation.clone(),
21293 }))
21294 .expect("file ingest should succeed");
21295
21296 assert_eq!(ingested.catalog.asset.study_id, study.study_id);
21297 assert_eq!(ingested.catalog.asset.asset_type, AssetType::File);
21298 let latest = ingested
21299 .catalog
21300 .latest_version
21301 .as_ref()
21302 .expect("ingest should create a latest version");
21303 assert_eq!(latest.version_label.as_deref(), Some("v1.0.0"));
21304 assert_eq!(ingested.datafile.datafile_id, latest.version_id);
21305 assert_eq!(ingested.datafile.compressed, Some(true));
21306 assert_eq!(ingested.datafile.encrypted, Some(true));
21307 assert_eq!(
21308 ingested.datafile.compression_algorithm.as_deref(),
21309 Some("zstd")
21310 );
21311 assert_eq!(
21312 ingested.datafile.encryption_algorithm.as_deref(),
21313 Some("AES-256-CBC with PBKDF2")
21314 );
21315 assert_eq!(
21316 ingested.datafile.encryption_key.as_ref().map(Vec::len),
21317 Some(32)
21318 );
21319 assert_eq!(ingested.datafile.edam_format, "EDAM:format_3752");
21320 assert!(ingested.datafile.storage_uri.starts_with("lake://"));
21321 assert_eq!(ingested.datafile.digest.as_str().len(), 64);
21322 assert_eq!(
21323 ingested.source_size_bytes,
21324 fs::metadata(&source_path)
21325 .expect("source metadata should read")
21326 .len()
21327 );
21328 assert!(ingested.stored_size_bytes > 0);
21329
21330 let stored_path = datafile_storage_uri_path(
21331 &lake_root.display().to_string(),
21332 &ingested.datafile.storage_uri,
21333 )
21334 .expect("storage URI should resolve to a path");
21335 assert!(stored_path.is_file());
21336 assert_eq!(
21337 stored_path.file_name().and_then(|name| name.to_str()),
21338 Some(
21339 format!(
21340 "p1_12_ingested_file_1_0_0_{}.zst",
21341 ingested.datafile.datafile_id.0.simple()
21342 )
21343 .as_str()
21344 )
21345 );
21346 assert_ne!(
21347 fs::read(&stored_path).expect("stored file should read"),
21348 fs::read(&source_path).expect("source file should read")
21349 );
21350 assert_eq!(ingested.lineage.transformation, transformation);
21351 assert!(ingested.lineage.inputs.is_empty());
21352 assert_eq!(
21353 ingested
21354 .lineage
21355 .outputs
21356 .iter()
21357 .map(|output| output.version_id)
21358 .collect::<Vec<_>>(),
21359 vec![latest.version_id]
21360 );
21361 let listed = block_on(service.list_datafiles(crate::ListDataFilesRequest {
21362 study: StudySelector::Id {
21363 study_id: study.study_id,
21364 },
21365 include_versions: true,
21366 }))
21367 .expect("Datafile list workflow should read the ingested catalogue");
21368 assert_eq!(listed.study_id, study.study_id);
21369 assert_eq!(listed.study_name, study.name);
21370 assert_eq!(listed.datafiles.len(), 1);
21371 assert_eq!(listed.datafiles[0].datafiles.len(), 1);
21372 assert_eq!(
21373 listed.datafiles[0].datafiles[0].datafile_id,
21374 ingested.datafile.datafile_id
21375 );
21376 assert_eq!(
21377 listed.datafiles[0].datafiles[0].edam_format,
21378 ingested.datafile.edam_format
21379 );
21380 assert_eq!(
21381 block_on(
21382 repositories
21383 .assets
21384 .list_datafiles_for_asset(ingested.catalog.asset.asset_id)
21385 )
21386 .expect("datafiles should list for asset"),
21387 vec![ingested.datafile]
21388 );
21389
21390 let _ = fs::remove_dir_all(&lake_root);
21391 }
21392
21393 #[test]
21394 fn ingest_file_provenance_failure_rolls_back_catalogue_and_managed_payload() {
21395 let repositories = AssetWorkflowRepositories::new();
21396 let service = repositories.service();
21397 let study = test_study("ticket_08_rollback_study");
21398 repositories.seed_study(study.clone());
21399 let source_path = workspace_file("data/ingest.csv");
21400 let lake_root = temp_lake_root_path();
21401 let asset_name = nc_name("ticket_08_rollback_file");
21402
21403 repositories.fail_next_transformation_outputs("ticket 08 injected provenance failure");
21404 let error = block_on(service.ingest_file(IngestFileRequest {
21405 classification: ahri_tre_types::IngestClassification::derived_high(),
21406 study_id: study.study_id,
21407 asset_id: None,
21408 asset_name: asset_name.clone(),
21409 version_id: None,
21410 source: IngestFileSource::LocalPath {
21411 path: source_path.display().to_string(),
21412 },
21413 lake_root: lake_root.display().to_string(),
21414 edam_format: Some("csv".to_string()),
21415 compress: true,
21416 encrypt: Some(false),
21417 description: Some("rollback fixture #ticket08rollback".to_string()),
21418 agent_instructions: None,
21419 version_note: None,
21420 transformation: test_transformation(
21421 180_008,
21422 TransformationType::Ingest,
21423 "ticket 08 failed provenance",
21424 ),
21425 }))
21426 .expect_err("injected provenance failure should fail ingest");
21427
21428 assert!(error.to_string().contains("injected provenance failure"));
21429 assert!(
21430 block_on(
21431 repositories
21432 .assets
21433 .get_asset_by_name(study.study_id, asset_name.as_str())
21434 )
21435 .expect("asset lookup should succeed")
21436 .is_none()
21437 );
21438 assert!(
21439 block_on(service.list_study_datafiles(ListStudyDataFilesRequest {
21440 study_id: study.study_id,
21441 include_versions: true,
21442 }))
21443 .expect("datafile catalogue should remain readable")
21444 .is_empty()
21445 );
21446 assert!(
21447 repositories
21448 .transformations
21449 .state
21450 .transformations
21451 .lock()
21452 .expect("transformation lock should not be poisoned")
21453 .is_empty()
21454 );
21455 assert!(
21456 repositories
21457 .state
21458 .tags
21459 .lock()
21460 .expect("tag lock should not be poisoned")
21461 .is_empty()
21462 );
21463 assert!(
21464 repositories
21465 .state
21466 .tag_links
21467 .lock()
21468 .expect("tag link lock should not be poisoned")
21469 .is_empty()
21470 );
21471 assert!(
21472 repositories
21473 .transformations
21474 .state
21475 .inputs
21476 .lock()
21477 .expect("transformation input lock should not be poisoned")
21478 .is_empty()
21479 );
21480 assert!(
21481 repositories
21482 .transformations
21483 .state
21484 .outputs
21485 .lock()
21486 .expect("transformation output lock should not be poisoned")
21487 .is_empty()
21488 );
21489 let mut pending = vec![lake_root.path().to_path_buf()];
21490 let mut files = Vec::new();
21491 while let Some(directory) = pending.pop() {
21492 if !directory.exists() {
21493 continue;
21494 }
21495 for entry in fs::read_dir(directory).expect("lake directory should be readable") {
21496 let path = entry.expect("lake entry should be readable").path();
21497 if path.is_dir() {
21498 pending.push(path);
21499 } else {
21500 files.push(path);
21501 }
21502 }
21503 }
21504 assert!(files.is_empty(), "failed ingest retained files: {files:?}");
21505 let _ = fs::remove_dir_all(&lake_root);
21506 }
21507
21508 #[test]
21509 fn ingest_file_metadata_failures_leave_no_partial_state() {
21510 #[derive(Clone, Copy, Debug)]
21511 enum FailurePoint {
21512 Asset,
21513 AssetTag,
21514 Version,
21515 Datafile,
21516 }
21517
21518 for failure in [
21519 FailurePoint::Asset,
21520 FailurePoint::AssetTag,
21521 FailurePoint::Version,
21522 FailurePoint::Datafile,
21523 ] {
21524 let repositories = AssetWorkflowRepositories::new();
21525 let service = repositories.service();
21526 let study = test_study(&format!("ticket_08_{failure:?}_study"));
21527 repositories.seed_study(study.clone());
21528 match failure {
21529 FailurePoint::Asset => repositories.fail_next_save_asset("asset failure"),
21530 FailurePoint::AssetTag => repositories.fail_next_attach_tag("tag failure"),
21531 FailurePoint::Version => {
21532 repositories.fail_next_save_asset_version("version failure")
21533 }
21534 FailurePoint::Datafile => repositories.fail_next_save_datafile("Datafile failure"),
21535 }
21536 let lake_root = temp_lake_root_path();
21537 let asset_name = nc_name(&format!("ticket_08_{failure:?}_file").to_ascii_lowercase());
21538
21539 block_on(service.ingest_file(IngestFileRequest {
21540 classification: ahri_tre_types::IngestClassification::derived_high(),
21541 study_id: study.study_id,
21542 asset_id: None,
21543 asset_name: asset_name.clone(),
21544 version_id: None,
21545 source: IngestFileSource::LocalPath {
21546 path: workspace_file("data/ingest.csv").display().to_string(),
21547 },
21548 lake_root: lake_root.display().to_string(),
21549 edam_format: Some("csv".to_string()),
21550 compress: false,
21551 encrypt: Some(false),
21552 description: Some("rollback fixture #ticket08stage".to_string()),
21553 agent_instructions: None,
21554 version_note: Some("rollback version #ticket08version".to_string()),
21555 transformation: test_transformation(
21556 180_020,
21557 TransformationType::Ingest,
21558 "ticket 08 staged failure",
21559 ),
21560 }))
21561 .expect_err("injected metadata failure should fail ingest");
21562
21563 assert!(
21564 block_on(
21565 repositories
21566 .assets
21567 .get_asset_by_name(study.study_id, asset_name.as_str())
21568 )
21569 .expect("asset lookup should succeed")
21570 .is_none(),
21571 "{failure:?} left an Asset"
21572 );
21573 assert!(
21574 repositories
21575 .state
21576 .versions_by_asset
21577 .lock()
21578 .expect("version lock should not be poisoned")
21579 .values()
21580 .all(Vec::is_empty),
21581 "{failure:?} left a version"
21582 );
21583 assert!(
21584 repositories
21585 .state
21586 .datafiles_by_version
21587 .lock()
21588 .expect("Datafile lock should not be poisoned")
21589 .is_empty(),
21590 "{failure:?} left a Datafile"
21591 );
21592 assert!(
21593 repositories
21594 .state
21595 .tags
21596 .lock()
21597 .expect("tag lock should not be poisoned")
21598 .is_empty(),
21599 "{failure:?} left a tag"
21600 );
21601 assert!(
21602 repositories
21603 .state
21604 .tag_links
21605 .lock()
21606 .expect("tag link lock should not be poisoned")
21607 .is_empty(),
21608 "{failure:?} left a tag link"
21609 );
21610 let mut pending = vec![lake_root.path().to_path_buf()];
21611 let mut files = Vec::new();
21612 while let Some(directory) = pending.pop() {
21613 if !directory.exists() {
21614 continue;
21615 }
21616 for entry in fs::read_dir(directory).expect("lake directory should be readable") {
21617 let path = entry.expect("lake entry should be readable").path();
21618 if path.is_dir() {
21619 pending.push(path);
21620 } else {
21621 files.push(path);
21622 }
21623 }
21624 }
21625 assert!(files.is_empty(), "{failure:?} left files: {files:?}");
21626 }
21627 }
21628
21629 #[test]
21630 fn cancelled_ingest_guard_removes_the_managed_payload_before_the_next_ingest() {
21631 let lake_root = temp_lake_root_path();
21632 let lake = DuckLakeAdapter::new(lake_root.path().to_string_lossy());
21633 let workflow_guard = block_on(Arc::clone(&FILE_INGEST_WORKFLOW_LOCK).lock_owned());
21634 let mut source = Cursor::new(b"id,value\n1,alpha\n".to_vec());
21635 let staged = lake
21636 .stage_datafile_stream(
21637 &StageDataFileStreamRequest {
21638 source_file_name: "cancelled.csv".to_string(),
21639 asset_id: AssetId(uuid::Uuid::new_v4()),
21640 version_id: VersionId(uuid::Uuid::new_v4()),
21641 study_id: StudyId(uuid::Uuid::new_v4()),
21642 asset_name: nc_name("cancelled_ingest"),
21643 major: 1,
21644 minor: 0,
21645 patch: 0,
21646 edam_format: "EDAM:format_3752".to_string(),
21647 compression: DataFileCompression::None,
21648 encryption: DataFileEncryption::None,
21649 },
21650 &mut source,
21651 )
21652 .expect("cancelled ingest fixture should stage");
21653 let guard = StagedFileGuard {
21654 staged: Some(staged),
21655 lake,
21656 workflow_guard: Some(workflow_guard),
21657 };
21658
21659 drop(guard);
21660 let next_guard = block_on(Arc::clone(&FILE_INGEST_WORKFLOW_LOCK).lock_owned());
21661 drop(next_guard);
21662
21663 let mut pending = vec![lake_root.path().to_path_buf()];
21664 let mut files = Vec::new();
21665 while let Some(directory) = pending.pop() {
21666 for entry in fs::read_dir(directory).expect("Lake directory should read") {
21667 let path = entry.expect("Lake entry should read").path();
21668 if path.is_dir() {
21669 pending.push(path);
21670 } else {
21671 files.push(path);
21672 }
21673 }
21674 }
21675 assert!(
21676 files.is_empty(),
21677 "cancelled ingest retained files: {files:?}"
21678 );
21679 }
21680
21681 #[test]
21682 fn cancelled_waiter_does_not_stop_owned_app_work() {
21683 let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
21684 let completed_by_worker = Arc::clone(&completed);
21685 let (started_sender, started_receiver) = std::sync::mpsc::channel();
21686 let (resume_sender, resume_receiver) = tokio::sync::oneshot::channel();
21687 let runtime = tokio::runtime::Builder::new_multi_thread()
21688 .worker_threads(2)
21689 .enable_all()
21690 .build()
21691 .expect("test Tokio runtime should build");
21692
21693 runtime.block_on(async move {
21694 let waiter = tokio::spawn(run_cancellation_safe_app_work(async move {
21695 started_sender
21696 .send(())
21697 .expect("worker start should be observed");
21698 resume_receiver.await.expect("worker should resume");
21699 completed_by_worker.store(true, std::sync::atomic::Ordering::Release);
21700 Ok::<(), AppError>(())
21701 }));
21702 tokio::task::spawn_blocking(move || {
21703 started_receiver.recv().expect("owned worker should start");
21704 })
21705 .await
21706 .expect("start wait should finish");
21707 waiter.abort();
21708 resume_sender.send(()).expect("owned worker should remain");
21709 });
21710
21711 for _ in 0..100 {
21712 if completed.load(std::sync::atomic::Ordering::Acquire) {
21713 return;
21714 }
21715 std::thread::sleep(StdDuration::from_millis(5));
21716 }
21717 panic!("owned app work stopped with its caller");
21718 }
21719
21720 #[test]
21721 fn common_datafile_ingest_rejects_bad_csv_and_cleans_trusted_scratch() {
21722 let repositories = AssetWorkflowRepositories::new();
21723 let service = repositories.service();
21724 let study = test_study("ticket_08_bad_content_study");
21725 repositories.seed_study(study.clone());
21726 let lake_root = temp_lake_root_path();
21727 let scratch_root = temp_lake_root_path();
21728 let session = test_lake_session_with_scratch(lake_root.path(), scratch_root.path());
21729 let asset_name = nc_name("ticket_08_bad_content_file");
21730
21731 let error = block_on(service.ingest_datafile(
21732 &session,
21733 IngestDataFileRequest {
21734 classification: ahri_tre_types::IngestClassification::derived_high(),
21735 study: StudySelector::Id {
21736 study_id: study.study_id,
21737 },
21738 asset_name: asset_name.clone(),
21739 content: IngestFileContent::new(
21740 "bad.csv",
21741 Some("text/csv".to_string()),
21742 Some(4),
21743 None,
21744 Cursor::new(vec![0xff, 0xfe, 0xfd, b'\n']),
21745 ),
21746 format: "csv".to_string(),
21747 description: None,
21748 new_version: false,
21749 bump_major: false,
21750 bump_minor: false,
21751 compress: false,
21752 encrypt: Some(false),
21753 },
21754 ))
21755 .expect_err("bad CSV must fail in the common workflow");
21756
21757 assert!(matches!(error, AppError::Validation(_)));
21758 assert!(
21759 session
21760 .scratch_attempt()
21761 .expect("test scratch should exist")
21762 .path()
21763 .read_dir()
21764 .expect("scratch should be readable")
21765 .next()
21766 .is_none()
21767 );
21768 assert!(
21769 block_on(
21770 repositories
21771 .assets
21772 .get_asset_by_name(study.study_id, asset_name.as_str())
21773 )
21774 .expect("asset lookup should succeed")
21775 .is_none()
21776 );
21777 assert!(
21778 lake_root
21779 .path()
21780 .read_dir()
21781 .expect("Lake should read")
21782 .next()
21783 .is_none(),
21784 "bad CSV left managed state"
21785 );
21786 }
21787
21788 #[test]
21789 fn session_local_file_ingest_rejects_bad_csv() {
21790 let repositories = AssetWorkflowRepositories::new();
21791 let service = repositories.service();
21792 let study = test_study("ticket_08_bad_local_study");
21793 repositories.seed_study(study.clone());
21794 let lake_root = temp_lake_root_path();
21795 let scratch_root = temp_lake_root_path();
21796 let source_root = temp_lake_root_path();
21797 fs::create_dir(source_root.path()).expect("source root should exist");
21798 let source_path = source_root.path().join("bad.csv");
21799 fs::write(&source_path, [0xff, 0xfe, 0xfd, b'\n']).expect("bad local CSV should write");
21800 let session = test_lake_session_with_scratch(lake_root.path(), scratch_root.path());
21801
21802 let error = block_on(service.ingest_file_for_session(
21803 &session,
21804 IngestFileRequest {
21805 classification: ahri_tre_types::IngestClassification::derived_high(),
21806 study_id: study.study_id,
21807 asset_id: None,
21808 asset_name: nc_name("ticket_08_bad_local_file"),
21809 version_id: None,
21810 source: IngestFileSource::LocalPath {
21811 path: source_path.display().to_string(),
21812 },
21813 lake_root: String::new(),
21814 edam_format: Some("csv".to_string()),
21815 compress: false,
21816 encrypt: Some(false),
21817 description: None,
21818 agent_instructions: None,
21819 version_note: None,
21820 transformation: test_transformation(
21821 180_030,
21822 TransformationType::Ingest,
21823 "ticket 08 bad local ingest",
21824 ),
21825 },
21826 ))
21827 .expect_err("bad local CSV must fail");
21828
21829 assert!(matches!(error, AppError::Validation(_)));
21830 assert!(
21831 session
21832 .scratch_attempt()
21833 .expect("test scratch should exist")
21834 .path()
21835 .read_dir()
21836 .expect("scratch should read")
21837 .next()
21838 .is_none()
21839 );
21840 assert!(
21841 lake_root
21842 .path()
21843 .read_dir()
21844 .expect("Lake should read")
21845 .next()
21846 .is_none()
21847 );
21848 }
21849
21850 #[test]
21851 fn common_datafile_ingest_has_the_same_meaning_for_file_and_upload_content() {
21852 let bytes = b"id,value\n1,alpha\n".to_vec();
21853 let mut meanings = Vec::new();
21854 for uploaded in [false, true] {
21855 let repositories = AssetWorkflowRepositories::new();
21856 let service = repositories.service();
21857 let study = test_study("ticket_08_parity_study");
21858 repositories.seed_study(study.clone());
21859 let lake_root = temp_lake_root_path();
21860 let scratch_root = temp_lake_root_path();
21861 let session = test_lake_session_with_scratch(lake_root.path(), scratch_root.path());
21862 let content = if uploaded {
21863 IngestFileContent::new(
21864 "baseline.csv",
21865 Some("text/csv".to_string()),
21866 Some(bytes.len() as u64),
21867 None,
21868 Cursor::new(bytes.clone()),
21869 )
21870 } else {
21871 let source_path = scratch_root.path().join("local-baseline.csv");
21872 fs::write(&source_path, &bytes).expect("local fixture should write");
21873 IngestFileContent::new(
21874 "baseline.csv",
21875 None,
21876 Some(bytes.len() as u64),
21877 None,
21878 fs::File::open(source_path).expect("local fixture should open"),
21879 )
21880 };
21881 let result = block_on(service.ingest_datafile(
21882 &session,
21883 IngestDataFileRequest {
21884 classification: ahri_tre_types::IngestClassification::derived_high(),
21885 study: StudySelector::Id {
21886 study_id: study.study_id,
21887 },
21888 asset_name: nc_name("baseline_csv"),
21889 content,
21890 format: "csv".to_string(),
21891 description: Some("Baseline CSV".to_string()),
21892 new_version: false,
21893 bump_major: false,
21894 bump_minor: false,
21895 compress: false,
21896 encrypt: Some(false),
21897 },
21898 ))
21899 .expect("common ingest should succeed");
21900 let version = result
21901 .catalog
21902 .latest_version
21903 .expect("latest version should exist");
21904 meanings.push((
21905 result.study_name,
21906 result.catalog.asset.name,
21907 result.catalog.asset.description,
21908 result.datafile.edam_format,
21909 result.datafile.digest,
21910 result.source_size_bytes,
21911 result.stored_size_bytes,
21912 (
21913 version.major,
21914 version.minor,
21915 version.patch,
21916 version.is_latest,
21917 ),
21918 ));
21919 }
21920
21921 assert_eq!(meanings[0], meanings[1]);
21922 }
21923
21924 #[test]
21925 fn ingest_file_repeat_uses_the_common_version_policy() {
21926 let repositories = AssetWorkflowRepositories::new();
21927 let service = repositories.service();
21928 let study = test_study("ticket_08_repeat_study");
21929 repositories.seed_study(study.clone());
21930 let source_path = workspace_file("data/ingest.csv");
21931 let lake_root = temp_lake_root_path();
21932 let asset_name = nc_name("ticket_08_repeat_file");
21933
21934 for sequence in [1, 2] {
21935 block_on(service.ingest_file(IngestFileRequest {
21936 classification: ahri_tre_types::IngestClassification::derived_high(),
21937 study_id: study.study_id,
21938 asset_id: None,
21939 asset_name: asset_name.clone(),
21940 version_id: None,
21941 source: IngestFileSource::LocalPath {
21942 path: source_path.display().to_string(),
21943 },
21944 lake_root: lake_root.display().to_string(),
21945 edam_format: Some("csv".to_string()),
21946 compress: false,
21947 encrypt: Some(false),
21948 description: Some("repeat fixture".to_string()),
21949 agent_instructions: None,
21950 version_note: None,
21951 transformation: test_transformation(
21952 180_008 + sequence,
21953 TransformationType::Ingest,
21954 "ticket 08 repeat ingest",
21955 ),
21956 }))
21957 .expect("repeat ingest should create the next governed version");
21958 }
21959
21960 let catalog = block_on(service.get_asset(GetAssetRequest {
21961 asset_id: None,
21962 study_id: Some(study.study_id),
21963 asset_name: Some(asset_name),
21964 }))
21965 .expect("repeat catalogue should read");
21966 assert_eq!(catalog.versions.len(), 2);
21967 let latest = catalog.latest_version.expect("latest version should exist");
21968 assert_eq!((latest.major, latest.minor, latest.patch), (1, 1, 0));
21969 let _ = fs::remove_dir_all(&lake_root);
21970 }
21971
21972 #[test]
21973 fn ingest_file_rejects_a_non_file_asset_without_new_state() {
21974 let repositories = AssetWorkflowRepositories::new();
21975 let service = repositories.service();
21976 let study = test_study("ticket_08_conflict_study");
21977 repositories.seed_study(study.clone());
21978 let asset = test_asset(study.study_id, "ticket_08_conflict_asset");
21979 block_on(service.create_asset(CreateAssetRequest {
21980 asset: asset.clone(),
21981 }))
21982 .expect("fixture Dataset asset should save");
21983 let lake_root = temp_lake_root_path();
21984
21985 let error = block_on(service.ingest_file(IngestFileRequest {
21986 classification: ahri_tre_types::IngestClassification::derived_high(),
21987 study_id: study.study_id,
21988 asset_id: None,
21989 asset_name: asset.name.clone(),
21990 version_id: None,
21991 source: IngestFileSource::LocalPath {
21992 path: workspace_file("data/ingest.csv").display().to_string(),
21993 },
21994 lake_root: lake_root.display().to_string(),
21995 edam_format: Some("csv".to_string()),
21996 compress: false,
21997 encrypt: Some(false),
21998 description: None,
21999 agent_instructions: None,
22000 version_note: None,
22001 transformation: test_transformation(
22002 180_011,
22003 TransformationType::Ingest,
22004 "ticket 08 conflict ingest",
22005 ),
22006 }))
22007 .expect_err("a Dataset asset must conflict with Datafile ingest");
22008
22009 assert!(matches!(error, AppError::Validation(_)));
22010 assert!(
22011 block_on(repositories.assets.list_asset_versions(asset.asset_id))
22012 .expect("fixture versions should list")
22013 .is_empty()
22014 );
22015 assert!(!lake_root.path().exists());
22016 }
22017
22018 #[test]
22019 fn ingest_file_fetches_https_fixture_without_persisting_source_uri() {
22020 let repositories = AssetWorkflowRepositories::new();
22021 let service = repositories.service();
22022 let study = test_study("https_ingest_success_study");
22023 repositories.seed_study(study.clone());
22024 let body = fs::read(workspace_file("data/ingest.csv")).expect("fixture CSV should read");
22025 let (base_url, _guard) = spawn_https_fixture_response("200 OK", &[], body);
22026 let lake_root = temp_lake_root_path();
22027 let uri = format!("{base_url}/fixture.csv?token=secret-token");
22028
22029 let ingested = block_on(service.ingest_file(IngestFileRequest {
22030 classification: ahri_tre_types::IngestClassification::derived_high(),
22031 study_id: study.study_id,
22032 asset_id: None,
22033 asset_name: nc_name("https_ingested_file"),
22034 version_id: None,
22035 source: IngestFileSource::HttpsUri { uri: uri.clone() },
22036 lake_root: lake_root.display().to_string(),
22037 edam_format: None,
22038 compress: true,
22039 encrypt: Some(false),
22040 description: Some("HTTPS fixture ingest".to_string()),
22041 agent_instructions: None,
22042 version_note: Some("HTTPS fixture".to_string()),
22043 transformation: test_transformation(
22044 180_001,
22045 TransformationType::Ingest,
22046 "ingest HTTPS fixture",
22047 ),
22048 }))
22049 .expect("HTTPS fixture ingest should succeed");
22050
22051 assert_eq!(ingested.catalog.asset.name.as_str(), "https_ingested_file");
22052 assert_eq!(ingested.datafile.edam_format, "EDAM:format_3752");
22053 assert!(ingested.datafile.storage_uri.starts_with("lake://"));
22054 let rendered =
22055 serde_json::to_string(&ingested).expect("ingested HTTPS payload should serialize");
22056 assert!(!rendered.contains("secret-token"));
22057 assert!(!rendered.contains(&uri));
22058
22059 let _ = fs::remove_dir_all(&lake_root);
22060 }
22061
22062 #[test]
22063 fn ingest_file_rejects_unsupported_uri_scheme_before_metadata() {
22064 let repositories = AssetWorkflowRepositories::new();
22065 let service = repositories.service();
22066 let study = test_study("https_ingest_unsupported_scheme_study");
22067 repositories.seed_study(study.clone());
22068 let lake_root = temp_lake_root_path();
22069
22070 let error = block_on(service.ingest_file(IngestFileRequest {
22071 classification: ahri_tre_types::IngestClassification::derived_high(),
22072 study_id: study.study_id,
22073 asset_id: None,
22074 asset_name: nc_name("unsupported_scheme_file"),
22075 version_id: None,
22076 source: IngestFileSource::HttpsUri {
22077 uri: "http://example.test/source.csv?token=secret".to_string(),
22078 },
22079 lake_root: lake_root.display().to_string(),
22080 edam_format: Some("csv".to_string()),
22081 compress: true,
22082 encrypt: Some(false),
22083 description: None,
22084 agent_instructions: None,
22085 version_note: None,
22086 transformation: test_transformation(
22087 180_002,
22088 TransformationType::Ingest,
22089 "unsupported URI ingest",
22090 ),
22091 }))
22092 .expect_err("unsupported scheme should fail");
22093
22094 assert!(
22095 error
22096 .to_string()
22097 .contains("unsupported ingest source URI scheme")
22098 );
22099 assert!(
22100 block_on(
22101 repositories
22102 .assets
22103 .get_asset_by_name(study.study_id, "unsupported_scheme_file")
22104 )
22105 .expect("asset lookup should succeed")
22106 .is_none()
22107 );
22108 }
22109
22110 #[test]
22111 fn ingest_file_https_failures_are_redacted_and_leave_no_metadata() {
22112 let cases = [
22113 (
22114 "status_failure_file",
22115 "403 Forbidden",
22116 vec![],
22117 Vec::new(),
22118 "HTTP status 403",
22119 None,
22120 ),
22121 (
22122 "oversized_file",
22123 "200 OK",
22124 vec![],
22125 b"too-large".to_vec(),
22126 "response limit",
22127 Some(4),
22128 ),
22129 (
22130 "empty_file",
22131 "200 OK",
22132 vec![],
22133 Vec::new(),
22134 "empty content",
22135 None,
22136 ),
22137 ];
22138
22139 for (asset_name, status, headers, body, expected, max_bytes) in cases {
22140 let repositories = AssetWorkflowRepositories::new();
22141 let service = repositories.service();
22142 let study = test_study(&format!("{asset_name}_study"));
22143 repositories.seed_study(study.clone());
22144 let (base_url, _guard) = spawn_https_fixture_response(status, &headers, body);
22145 if let Some(max_bytes) = max_bytes {
22146 *TEST_HTTPS_MAX_BYTES
22147 .lock()
22148 .expect("test HTTPS max bytes lock should not be poisoned") = Some(max_bytes);
22149 }
22150 let lake_root = temp_lake_root_path();
22151 let uri = format!("{base_url}/source.csv?token=secret-token");
22152
22153 let error = block_on(service.ingest_file(IngestFileRequest {
22154 classification: ahri_tre_types::IngestClassification::derived_high(),
22155 study_id: study.study_id,
22156 asset_id: None,
22157 asset_name: nc_name(asset_name),
22158 version_id: None,
22159 source: IngestFileSource::HttpsUri { uri },
22160 lake_root: lake_root.display().to_string(),
22161 edam_format: Some("csv".to_string()),
22162 compress: true,
22163 encrypt: Some(false),
22164 description: None,
22165 agent_instructions: None,
22166 version_note: None,
22167 transformation: test_transformation(
22168 180_003,
22169 TransformationType::Ingest,
22170 "failed HTTPS ingest",
22171 ),
22172 }))
22173 .expect_err("HTTPS failure should fail");
22174 let message = error.to_string();
22175 assert!(message.contains(expected), "{message}");
22176 assert!(!message.contains("secret-token"), "{message}");
22177 assert!(
22178 block_on(
22179 repositories
22180 .assets
22181 .get_asset_by_name(study.study_id, asset_name)
22182 )
22183 .expect("asset lookup should succeed")
22184 .is_none()
22185 );
22186 let _ = fs::remove_dir_all(&lake_root);
22187 }
22188 }
22189
22190 #[test]
22191 fn ingest_file_https_timeout_is_redacted_and_leaves_no_metadata() {
22192 let repositories = AssetWorkflowRepositories::new();
22193 let service = repositories.service();
22194 let study = test_study("https_ingest_timeout_study");
22195 repositories.seed_study(study.clone());
22196 let (base_url, _guard) = spawn_https_fixture_response_with_delay(
22197 "200 OK",
22198 &[],
22199 b"id,value\n1,slow\n".to_vec(),
22200 Some(StdDuration::from_millis(200)),
22201 );
22202 *TEST_HTTPS_TIMEOUT
22203 .lock()
22204 .expect("test HTTPS timeout lock should not be poisoned") =
22205 Some(StdDuration::from_millis(25));
22206 let lake_root = temp_lake_root_path();
22207
22208 let error = block_on(service.ingest_file(IngestFileRequest {
22209 classification: ahri_tre_types::IngestClassification::derived_high(),
22210 study_id: study.study_id,
22211 asset_id: None,
22212 asset_name: nc_name("timeout_file"),
22213 version_id: None,
22214 source: IngestFileSource::HttpsUri {
22215 uri: format!("{base_url}/source.csv?token=secret-token"),
22216 },
22217 lake_root: lake_root.display().to_string(),
22218 edam_format: Some("csv".to_string()),
22219 compress: true,
22220 encrypt: Some(false),
22221 description: None,
22222 agent_instructions: None,
22223 version_note: None,
22224 transformation: test_transformation(
22225 180_004,
22226 TransformationType::Ingest,
22227 "timeout HTTPS ingest",
22228 ),
22229 }))
22230 .expect_err("HTTPS timeout should fail");
22231 let message = error.to_string();
22232 assert!(!message.contains("secret-token"), "{message}");
22233 assert!(
22234 block_on(
22235 repositories
22236 .assets
22237 .get_asset_by_name(study.study_id, "timeout_file")
22238 )
22239 .expect("asset lookup should succeed")
22240 .is_none()
22241 );
22242 let _ = fs::remove_dir_all(&lake_root);
22243 }
22244
22245 #[test]
22246 fn ingest_file_redirect_policy_rejects_unsupported_targets_and_limit() {
22247 let https_url = Url::parse("https://example.test/next.csv").unwrap();
22248 let http_url = Url::parse("http://example.test/next.csv").unwrap();
22249
22250 assert!(validate_https_ingest_redirect(0, &https_url).is_ok());
22251 assert_eq!(
22252 validate_https_ingest_redirect(0, &http_url),
22253 Err("HTTPS ingest redirect target uses an unsupported scheme")
22254 );
22255 assert_eq!(
22256 validate_https_ingest_redirect(HTTPS_INGEST_MAX_REDIRECTS, &https_url),
22257 Err("HTTPS ingest redirect limit exceeded")
22258 );
22259 }
22260
22261 #[test]
22262 fn file_to_dataset_loads_external_file_into_ducklake_with_ingest_provenance() {
22263 let repositories = AssetWorkflowRepositories::new();
22264 let service = repositories.service();
22265 let study = test_study("p1_13_file_dataset_study");
22266 repositories.seed_study(study.clone());
22267 let source_path = workspace_file("data/ingest.csv");
22268 assert!(
22269 source_path.is_file(),
22270 "P1.13 ingest fixture should exist at {}",
22271 source_path.display()
22272 );
22273 let lake_root = temp_lake_root_path();
22274 let mut session = test_lake_session(&lake_root);
22275 let transformation =
22276 test_transformation(96, TransformationType::Ingest, "ingest csv file to dataset");
22277
22278 let materialized = block_on(service.file_to_dataset(
22279 &mut session,
22280 FileToDatasetRequest {
22281 classification: ahri_tre_types::IngestClassification::derived_high(),
22282 study_id: study.study_id,
22283 dataset_asset_id: None,
22284 dataset_name: nc_name("p1_13_file_dataset"),
22285 dataset_version_id: None,
22286 input_path: source_path.display().to_string(),
22287 input_format: Some("csv".to_string()),
22288 header: None,
22289 delimiter: None,
22290 null_strings: Vec::new(),
22291 json_format: None,
22292 sheet: None,
22293 description: Some("Dataset loaded from an external file".to_string()),
22294 agent_instructions: Some("Treat as a governed dataset".to_string()),
22295 version_note: Some("Initial file dataset load".to_string()),
22296 transformation: transformation.clone(),
22297 variable_registrations: vec![
22298 DatasetVariableRegistrationRequest {
22299 variable: test_variable(10_001, 90_001, "country", KeyRole::Record, None),
22300 row_role: Some(KeyRole::Record),
22301 vocabulary: None,
22302 vocabulary_items: Vec::new(),
22303 },
22304 DatasetVariableRegistrationRequest {
22305 variable: test_variable(
22306 10_002,
22307 90_001,
22308 "category",
22309 KeyRole::None,
22310 Some(VocabularyId(20_001)),
22311 ),
22312 row_role: None,
22313 vocabulary: Some(test_vocabulary(20_001, 90_001, "category_codes")),
22314 vocabulary_items: vec![
22315 test_vocabulary_item(30_001, 20_001, 1, "A"),
22316 test_vocabulary_item(30_002, 20_001, 2, "C"),
22317 ],
22318 },
22319 ],
22320 },
22321 ))
22322 .expect("file-to-dataset should succeed");
22323
22324 assert_eq!(materialized.catalog.asset.study_id, study.study_id);
22325 assert_eq!(materialized.catalog.asset.asset_type, AssetType::Dataset);
22326 assert_eq!(materialized.row_count, 10);
22327 assert_eq!(materialized.column_count, 5);
22328 assert!(materialized.lake_schema.starts_with("study_"));
22329 assert_eq!(materialized.lake_table, "p1_13_file_dataset_1_0_0");
22330 assert_eq!(
22331 materialized.lineage.transformation.transformation_type,
22332 TransformationType::Ingest
22333 );
22334 assert!(materialized.lineage.inputs.is_empty());
22335 assert_eq!(
22336 materialized
22337 .lineage
22338 .outputs
22339 .iter()
22340 .map(|output| output.version_id)
22341 .collect::<Vec<_>>(),
22342 vec![materialized.dataset.dataset_id]
22343 );
22344 assert_eq!(materialized.variables.len(), 2);
22345 assert_eq!(materialized.variables[0].variable.name.as_str(), "country");
22346 assert_eq!(materialized.variables[0].link.row_role, KeyRole::Record);
22347 assert_eq!(materialized.variables[1].variable.name.as_str(), "category");
22348 assert_eq!(materialized.variables[1].link.row_role, KeyRole::None);
22349 assert_eq!(
22350 materialized.variables[1]
22351 .vocabulary
22352 .as_ref()
22353 .map(|vocabulary| vocabulary.name.as_str()),
22354 Some("category_codes")
22355 );
22356 assert_eq!(materialized.variables[1].vocabulary_items.len(), 2);
22357 assert_eq!(
22358 block_on(
22359 repositories
22360 .variables
22361 .list_dataset_variables(materialized.dataset.dataset_id)
22362 )
22363 .expect("dataset variables should list"),
22364 materialized
22365 .variables
22366 .iter()
22367 .map(|registered| registered.link.clone())
22368 .collect::<Vec<_>>()
22369 );
22370 assert_eq!(
22371 block_on(
22372 repositories
22373 .assets
22374 .list_datasets_for_asset(materialized.catalog.asset.asset_id)
22375 )
22376 .expect("datasets should list for asset"),
22377 vec![materialized.dataset.clone()]
22378 );
22379 assert_eq!(
22380 dataset_row_count(
22381 &session,
22382 &materialized.lake_schema,
22383 &materialized.lake_table
22384 ),
22385 10
22386 );
22387
22388 let _ = fs::remove_dir_all(&lake_root);
22389 }
22390
22391 #[test]
22392 fn live_failed_file_materialization_rolls_back_metadata() {
22393 use futures::executor::block_on;
22394 if std::env::var("RUN_LIVE_PGMETA_SMOKE").ok().as_deref() != Some("true") {
22395 return;
22396 }
22397 let ca = fs::read_to_string(std::env::var("PGSSLROOTCERT").unwrap()).unwrap();
22398 let adapter = ahri_tre_pgmeta::PgMetadataAdapter::direct_with_ca(
22399 ahri_tre_pgmeta::DirectPostgresConnectConfig {
22400 host: std::env::var("TRE_SERVER").unwrap(),
22401 port: std::env::var("PGPORT").unwrap().parse().unwrap(),
22402 dbname: std::env::var("TRE_DBNAME").unwrap(),
22403 username: std::env::var("TRE_USER").unwrap(),
22404 password: Some(std::env::var("TRE_PWD").unwrap()),
22405 sslmode: Some("verify-full".into()),
22406 connect_timeout_secs: Some(10),
22407 },
22408 vec![ca],
22409 );
22410 let connection = adapter.connect().unwrap();
22411 let lake_root = temp_lake_root_path();
22412 let _fixture_session = test_lake_session(&lake_root);
22413 let executor = ahri_tre_lake::DatasetExecutor::acquire(
22414 &lake_root.with_extension("scratch"),
22415 uuid::Uuid::new_v4(),
22416 )
22417 .unwrap();
22418 let mut session = DataStoreSession::new(
22419 test_runtime(&lake_root),
22420 crate::StoreSessionConnection::Direct(Arc::new(Mutex::new(connection))),
22421 duckdb::Connection::open_in_memory().unwrap(),
22422 );
22423 session = session.with_dataset_executor(executor.clone());
22424 let source_attempt = ahri_tre_lake::TrustedScratch::open(
22425 &lake_root.with_extension("scratch"),
22426 [lake_root.as_ref()],
22427 )
22428 .unwrap()
22429 .create_attempt(
22430 ahri_tre_lake::ScratchAttemptId::new(&uuid::Uuid::new_v4().simple().to_string())
22431 .unwrap(),
22432 )
22433 .unwrap();
22434 session = session.with_scratch_attempt(Some(source_attempt));
22435 let service = AppService::from_datastore_session(&session).unwrap();
22436 let mut study = test_study(&format!("admission_{}", uuid::Uuid::new_v4().simple()));
22437 study.created_by = None;
22438 let study = block_on(
22439 service
22440 .registrations
22441 .save_study_registration(&study, &[], &[]),
22442 )
22443 .unwrap();
22444 let version_id = ahri_tre_types::VersionId(uuid::Uuid::new_v4());
22445 let request = FileToDatasetRequest {
22446 classification: ahri_tre_types::IngestClassification::derived_high(),
22447 study_id: study.study_id,
22448 dataset_asset_id: None,
22449 dataset_name: nc_name("rollback_dataset"),
22450 dataset_version_id: Some(version_id),
22451 input_path: workspace_file("data/ingest.csv").display().to_string(),
22452 input_format: Some("csv".into()),
22453 header: None,
22454 delimiter: None,
22455 null_strings: Vec::new(),
22456 json_format: None,
22457 sheet: None,
22458 description: Some("rollback #admission".into()),
22459 agent_instructions: None,
22460 version_note: None,
22461 transformation: test_transformation(
22462 404,
22463 TransformationType::Ingest,
22464 "failed admission",
22465 ),
22466 variable_registrations: vec![DatasetVariableRegistrationRequest {
22467 variable: test_variable(94104, i64::from(i32::MAX), "country", KeyRole::None, None),
22468 row_role: None,
22469 vocabulary: None,
22470 vocabulary_items: Vec::new(),
22471 }],
22472 };
22473 let competing_scratch = ahri_tre_lake::TrustedScratch::open(
22474 &lake_root.with_extension("scratch"),
22475 [lake_root.as_ref()],
22476 )
22477 .unwrap()
22478 .create_attempt(
22479 ahri_tre_lake::ScratchAttemptId::new(&uuid::Uuid::new_v4().simple().to_string())
22480 .unwrap(),
22481 )
22482 .unwrap();
22483 let mut competing_session = DataStoreSession::new(
22484 test_runtime(&lake_root),
22485 crate::StoreSessionConnection::Direct(Arc::new(Mutex::new(adapter.connect().unwrap()))),
22486 duckdb::Connection::open_in_memory().unwrap(),
22487 )
22488 .with_scratch_attempt(Some(competing_scratch))
22489 .with_dataset_executor(executor);
22490 let competing_service = AppService::from_datastore_session(&competing_session).unwrap();
22491 let prepare = |asset_id| PrepareDatasetVersionRequest {
22492 output_intent: None,
22493 inherit_risk: false,
22494 classification: ahri_tre_types::IngestClassification::derived_high(),
22495 study_id: study.study_id,
22496 dataset_asset_id: asset_id,
22497 dataset_name: request.dataset_name.clone(),
22498 dataset_version_id: None,
22499 description: request.description.clone(),
22500 agent_instructions: None,
22501 version_note: None,
22502 created_by: None,
22503 };
22504 let pending_tag = format!("pending_{}", uuid::Uuid::new_v4().simple());
22505 let rejected_acceptance = block_on(service.prepare_dataset_attempt_with_acceptance(
22506 &session,
22507 prepare(None),
22508 |repository, attempt| {
22509 assert!(attempt.version.is_none(), "acceptance precedes allocation");
22510 assert!(!attempt.creation_intent);
22511 futures::FutureExt::now_or_never(repository.create_tag(&NewTagRecord {
22512 name: pending_tag.clone(),
22513 }))
22514 .unwrap()?;
22515 Err(CoreError::Validation("fixture acceptance rejection".into()))
22516 },
22517 ));
22518 assert!(
22519 matches!(rejected_acceptance, Err(error) if error.to_string().contains("fixture acceptance rejection"))
22520 );
22521 assert!(
22522 !block_on(service.tags.list_tags())
22523 .unwrap()
22524 .iter()
22525 .any(|tag| tag.name == pending_tag)
22526 );
22527 assert!(
22528 block_on(
22529 service
22530 .assets
22531 .get_asset_by_name(study.study_id, "rollback_dataset")
22532 )
22533 .unwrap()
22534 .is_none()
22535 );
22536 let held = block_on(service.prepare_dataset_attempt(&session, prepare(None))).unwrap();
22539 let mut valid_competitor = request.clone();
22540 valid_competitor.variable_registrations.clear();
22541 valid_competitor.transformation.created_by = None;
22542 assert!(matches!(
22543 block_on(
22544 competing_service.file_to_dataset(&mut competing_session, valid_competitor.clone())
22545 ),
22546 Err(AppError::Conflict(_))
22547 ));
22548 held.abandon(&mut session).unwrap();
22549 let error = block_on(service.file_to_dataset(&mut session, request.clone()))
22550 .expect_err("missing Domain must reject variable admission");
22551 assert!(error.to_string().contains("dataset variable"), "{error}");
22552 assert!(
22553 block_on(
22554 service
22555 .assets
22556 .get_asset_by_name(study.study_id, "rollback_dataset")
22557 )
22558 .unwrap()
22559 .is_none(),
22560 "failed materialization must roll back its Asset through the real adapter"
22561 );
22562 assert!(
22563 block_on(service.assets.get_asset_version(version_id))
22564 .unwrap()
22565 .is_none()
22566 );
22567 assert!(
22568 block_on(service.assets.get_dataset(version_id))
22569 .unwrap()
22570 .is_none()
22571 );
22572
22573 assert!(!dataset_table_exists(
22574 &session,
22575 &format!("study_{}", study.study_id.0.to_string().replace('-', "_")),
22576 "rollback_dataset_1_0_0"
22577 ));
22578 let mut successful = request.clone();
22579 successful.dataset_version_id = None;
22580 successful.variable_registrations.clear();
22581 successful.transformation.created_by = None;
22582 let admitted = block_on(service.file_to_dataset(&mut session, successful)).unwrap();
22583 let held = block_on(
22584 service
22585 .prepare_dataset_attempt(&session, prepare(Some(admitted.catalog.asset.asset_id))),
22586 )
22587 .unwrap();
22588 assert_eq!(
22589 (
22590 held.prepared.version.major,
22591 held.prepared.version.minor,
22592 held.prepared.version.patch
22593 ),
22594 (1, 1, 0)
22595 );
22596 valid_competitor.dataset_asset_id = Some(admitted.catalog.asset.asset_id);
22597 assert!(matches!(
22598 block_on(competing_service.file_to_dataset(&mut competing_session, valid_competitor)),
22599 Err(AppError::Conflict(_))
22600 ));
22601 held.abandon(&mut session).unwrap();
22602 let mut failed = request.clone();
22603 failed.dataset_version_id = Some(VersionId(uuid::Uuid::new_v4()));
22604 failed.transformation.created_by = None;
22605 let failed_id = failed.dataset_version_id.unwrap();
22606 assert!(block_on(service.file_to_dataset(&mut session, failed)).is_err());
22607 let catalog = block_on(service.get_asset(GetAssetRequest {
22608 asset_id: Some(admitted.catalog.asset.asset_id),
22609 study_id: None,
22610 asset_name: None,
22611 }))
22612 .unwrap();
22613 assert_eq!(
22614 catalog.latest_version.unwrap().version_id,
22615 admitted.dataset.dataset_id
22616 );
22617 assert_eq!(catalog.versions.len(), 1);
22618 assert_eq!(
22619 dataset_row_count(&session, &admitted.lake_schema, &admitted.lake_table),
22620 10
22621 );
22622 assert!(!dataset_table_exists(
22623 &session,
22624 &admitted.lake_schema,
22625 "rollback_dataset_1_1_0"
22626 ));
22627 assert!(
22628 session
22629 .dataset_admission_repository()
22630 .unwrap()
22631 .dataset_admission_receipt(failed_id)
22632 .unwrap()
22633 .is_none()
22634 );
22635 let mut attempt = block_on(
22639 service
22640 .prepare_dataset_attempt(&session, prepare(Some(admitted.catalog.asset.asset_id))),
22641 )
22642 .unwrap();
22643 let completion_version = attempt.prepared.version.version_id;
22644 let lake = DuckLakeAdapter::new(&session.runtime().lake.data_path);
22645 let loaded = lake
22646 .load_dataset_table_from_file_reserved(
22647 session.lake_connection(),
22648 &mut attempt.authority,
22649 &LoadDatasetTableFromFileRequest {
22650 study_id: study.study_id,
22651 dataset_name: request.dataset_name.clone(),
22652 major: 1,
22653 minor: 1,
22654 patch: 0,
22655 source_path: workspace_file("data/ingest.csv"),
22656 format: DatasetFileFormat::Csv,
22657 options: DatasetFileReadOptions::default(),
22658 },
22659 )
22660 .unwrap();
22661 let mut transformation = request.transformation.clone();
22662 transformation.created_by = None;
22663 let completion_tag = format!("completion_{}", uuid::Uuid::new_v4().simple());
22664 let mut observed_completion = false;
22665 let outcome = block_on(service.save_dataset_metadata_with_completion(
22666 &session,
22667 &mut attempt,
22668 DatasetAdmissionMetadata {
22669 dataset: ahri_tre_types::DatasetRecord {
22670 dataset_id: completion_version,
22671 },
22672 transformation: &transformation,
22673 input_version_ids: &[],
22674 registration: DatasetMetadataRegistration {
22675 redcap_dictionary: None,
22676 variable_registrations: Vec::new(),
22677 loaded_column_names: Some(&loaded.column_names),
22678 force_metadata_updates: false,
22679 },
22680 },
22681 |repository, records| {
22682 assert_eq!(records.1.dataset_id, completion_version);
22683 assert_eq!(records.2.outputs[0].version_id, completion_version);
22684 futures::FutureExt::now_or_never(repository.create_tag(&NewTagRecord {
22685 name: completion_tag.clone(),
22686 }))
22687 .unwrap()
22688 .unwrap();
22689 observed_completion = true;
22690 Err(AppError::Infrastructure(
22691 "fixture completion rejection".into(),
22692 ))
22693 },
22694 ));
22695 assert!(observed_completion);
22696 let error = attempt.finish(&mut session, outcome).unwrap_err();
22697 assert!(error.to_string().contains("fixture completion rejection"));
22698 assert!(
22699 block_on(service.assets.get_asset_version(completion_version))
22700 .unwrap()
22701 .is_none()
22702 );
22703 assert!(
22704 block_on(service.assets.get_dataset(completion_version))
22705 .unwrap()
22706 .is_none()
22707 );
22708 assert!(
22709 !block_on(service.tags.list_tags())
22710 .unwrap()
22711 .iter()
22712 .any(|tag| tag.name == completion_tag)
22713 );
22714 let catalogue = block_on(service.get_asset(GetAssetRequest {
22715 asset_id: Some(admitted.catalog.asset.asset_id),
22716 study_id: None,
22717 asset_name: None,
22718 }))
22719 .unwrap();
22720 assert_eq!(
22721 catalogue.latest_version.unwrap().version_id,
22722 admitted.dataset.dataset_id
22723 );
22724 assert_eq!(catalogue.versions.len(), 1);
22725 assert!(!dataset_table_exists(
22726 &session,
22727 &loaded.relation.schema_name,
22728 &loaded.relation.table_name
22729 ));
22730 assert!(
22731 session
22732 .dataset_admission_repository()
22733 .unwrap()
22734 .dataset_admission_receipt(completion_version)
22735 .unwrap()
22736 .is_none()
22737 );
22738 block_on(
22741 service
22742 .prepare_dataset_attempt(&session, prepare(Some(admitted.catalog.asset.asset_id))),
22743 )
22744 .unwrap()
22745 .abandon(&mut session)
22746 .unwrap();
22747 if let Some(mut connection) = session.direct_store_connection() {
22748 connection.client().batch_execute("DELETE FROM public.dataset_executor_generation WHERE NOT EXISTS (SELECT 1 FROM public.dataset_output_reservations)").unwrap();
22749 }
22750 block_on(service.studies.delete_study_record(study.study_id)).unwrap();
22751 }
22752
22753 #[test]
22754 fn failed_file_to_dataset_after_lake_load_has_no_complete_latest() {
22755 let repositories = AssetWorkflowRepositories::new();
22756 let service = repositories.service();
22757 let study = test_study("issue04_failed_first_dataset_study");
22758 repositories.seed_study(study.clone());
22759 let source_path = workspace_file("data/ingest.csv");
22760 let lake_root = temp_lake_root_path();
22761 let mut session = test_lake_session(&lake_root);
22762
22763 repositories.fail_next_save_dataset("issue04 injected save_dataset failure");
22764 let error = block_on(service.file_to_dataset(
22765 &mut session,
22766 FileToDatasetRequest {
22767 classification: ahri_tre_types::IngestClassification::derived_high(),
22768 study_id: study.study_id,
22769 dataset_asset_id: None,
22770 dataset_name: nc_name("issue04_failed_first_dataset"),
22771 dataset_version_id: None,
22772 input_path: source_path.display().to_string(),
22773 input_format: Some("csv".to_string()),
22774 header: None,
22775 delimiter: None,
22776 null_strings: Vec::new(),
22777 json_format: None,
22778 sheet: None,
22779 description: Some("Dataset that will fail".to_string()),
22780 agent_instructions: None,
22781 version_note: Some("Failed first materialization #attempt".to_string()),
22782 transformation: test_transformation(
22783 404,
22784 TransformationType::Ingest,
22785 "issue04 failed first file materialization",
22786 ),
22787 variable_registrations: Vec::new(),
22788 },
22789 ))
22790 .expect_err("injected metadata failure should fail materialization");
22791 assert!(
22792 error
22793 .to_string()
22794 .contains("issue04 injected save_dataset failure"),
22795 "{error}"
22796 );
22797
22798 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
22799 study_id: study.study_id,
22800 include_versions: false,
22801 }))
22802 .expect("dataset list should still be readable");
22803 assert!(list.datasets.is_empty());
22804
22805 let metadata_error = block_on(service.get_study_dataset(GetStudyDatasetRequest {
22806 study_id: study.study_id,
22807 dataset_name: nc_name("issue04_failed_first_dataset"),
22808 with_variables: true,
22809 }))
22810 .expect_err("dataset metadata should report no complete version");
22811 assert!(
22812 metadata_error.to_string().contains("no complete versions"),
22813 "{metadata_error}"
22814 );
22815 assert!(!dataset_table_exists(
22816 &session,
22817 "study_issue04_failed_first_dataset_study",
22818 "issue04_failed_first_dataset_1_0_0"
22819 ));
22820
22821 let _ = fs::remove_dir_all(&lake_root);
22822 }
22823
22824 #[test]
22825 fn failed_dataset_variable_link_does_not_create_complete_latest() {
22826 let repositories = AssetWorkflowRepositories::new();
22827 let service = repositories.service();
22828 let study = test_study("issue04_failed_variable_link_study");
22829 let domain = test_domain(94_004, "issue04_failed_variable_link_domain");
22830 repositories.seed_study(study.clone());
22831 repositories.seed_domain(domain.clone());
22832 let source_path = workspace_file("data/ingest.csv");
22833 let lake_root = temp_lake_root_path();
22834 let mut session = test_lake_session(&lake_root);
22835
22836 repositories.fail_next_link_dataset_variable("issue04 injected dataset-variable failure");
22837 let error = block_on(service.file_to_dataset(
22838 &mut session,
22839 FileToDatasetRequest {
22840 classification: ahri_tre_types::IngestClassification::derived_high(),
22841 study_id: study.study_id,
22842 dataset_asset_id: None,
22843 dataset_name: nc_name("issue04_failed_variable_link_dataset"),
22844 dataset_version_id: None,
22845 input_path: source_path.display().to_string(),
22846 input_format: Some("csv".to_string()),
22847 header: None,
22848 delimiter: None,
22849 null_strings: Vec::new(),
22850 json_format: None,
22851 sheet: None,
22852 description: Some("Dataset with failed variable link".to_string()),
22853 agent_instructions: None,
22854 version_note: Some("Failed variable link materialization".to_string()),
22855 transformation: test_transformation(
22856 407,
22857 TransformationType::Ingest,
22858 "issue04 failed variable link materialization",
22859 ),
22860 variable_registrations: vec![DatasetVariableRegistrationRequest {
22861 variable: test_variable(
22862 94_104,
22863 domain.domain_id.0,
22864 "country",
22865 KeyRole::None,
22866 None,
22867 ),
22868 row_role: None,
22869 vocabulary: None,
22870 vocabulary_items: Vec::new(),
22871 }],
22872 },
22873 ))
22874 .expect_err("injected dataset-variable failure should fail");
22875 assert!(
22876 error
22877 .to_string()
22878 .contains("issue04 injected dataset-variable failure"),
22879 "{error}"
22880 );
22881
22882 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
22883 study_id: study.study_id,
22884 include_versions: false,
22885 }))
22886 .expect("dataset list should be readable");
22887 assert!(list.datasets.is_empty());
22888 assert!(!dataset_table_exists(
22889 &session,
22890 "study_issue04_failed_variable_link_study",
22891 "issue04_failed_variable_link_dataset_1_0_0"
22892 ));
22893
22894 let _ = fs::remove_dir_all(&lake_root);
22895 }
22896
22897 #[test]
22898 fn failed_tag_attachment_does_not_create_complete_latest() {
22899 let repositories = AssetWorkflowRepositories::new();
22900 let service = repositories.service();
22901 let study = test_study("issue04_failed_tag_study");
22902 repositories.seed_study(study.clone());
22903 let source_path = workspace_file("data/ingest.csv");
22904 let lake_root = temp_lake_root_path();
22905 let mut session = test_lake_session(&lake_root);
22906
22907 repositories.fail_next_attach_tag("issue04 injected tag failure");
22908 let error = block_on(service.file_to_dataset(
22909 &mut session,
22910 FileToDatasetRequest {
22911 classification: ahri_tre_types::IngestClassification::derived_high(),
22912 study_id: study.study_id,
22913 dataset_asset_id: None,
22914 dataset_name: nc_name("issue04_failed_tag_dataset"),
22915 dataset_version_id: None,
22916 input_path: source_path.display().to_string(),
22917 input_format: Some("csv".to_string()),
22918 header: None,
22919 delimiter: None,
22920 null_strings: Vec::new(),
22921 json_format: None,
22922 sheet: None,
22923 description: Some("Dataset with failed #tag".to_string()),
22924 agent_instructions: None,
22925 version_note: Some("Failed tag materialization".to_string()),
22926 transformation: test_transformation(
22927 409,
22928 TransformationType::Ingest,
22929 "issue04 failed tag materialization",
22930 ),
22931 variable_registrations: Vec::new(),
22932 },
22933 ))
22934 .expect_err("injected tag failure should fail");
22935 assert!(
22936 error.to_string().contains("issue04 injected tag failure"),
22937 "{error}"
22938 );
22939
22940 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
22941 study_id: study.study_id,
22942 include_versions: false,
22943 }))
22944 .expect("dataset list should be readable");
22945 assert!(list.datasets.is_empty());
22946 assert!(!dataset_table_exists(
22947 &session,
22948 "study_issue04_failed_tag_study",
22949 "issue04_failed_tag_dataset_1_0_0"
22950 ));
22951
22952 let _ = fs::remove_dir_all(&lake_root);
22953 }
22954
22955 #[test]
22956 fn failed_vocabulary_registration_does_not_create_complete_latest() {
22957 let repositories = AssetWorkflowRepositories::new();
22958 let service = repositories.service();
22959 let study = test_study("issue04_failed_vocabulary_study");
22960 let domain = test_domain(94_005, "issue04_failed_vocabulary_domain");
22961 repositories.seed_study(study.clone());
22962 repositories.seed_domain(domain.clone());
22963 let source_path = workspace_file("data/ingest.csv");
22964 let lake_root = temp_lake_root_path();
22965 let mut session = test_lake_session(&lake_root);
22966
22967 repositories.fail_next_create_vocabulary("issue04 injected vocabulary failure");
22968 let error = block_on(service.file_to_dataset(
22969 &mut session,
22970 FileToDatasetRequest {
22971 classification: ahri_tre_types::IngestClassification::derived_high(),
22972 study_id: study.study_id,
22973 dataset_asset_id: None,
22974 dataset_name: nc_name("issue04_failed_vocabulary_dataset"),
22975 dataset_version_id: None,
22976 input_path: source_path.display().to_string(),
22977 input_format: Some("csv".to_string()),
22978 header: None,
22979 delimiter: None,
22980 null_strings: Vec::new(),
22981 json_format: None,
22982 sheet: None,
22983 description: Some("Dataset with failed vocabulary".to_string()),
22984 agent_instructions: None,
22985 version_note: Some("Failed vocabulary materialization".to_string()),
22986 transformation: test_transformation(
22987 410,
22988 TransformationType::Ingest,
22989 "issue04 failed vocabulary materialization",
22990 ),
22991 variable_registrations: vec![DatasetVariableRegistrationRequest {
22992 variable: test_variable(
22993 94_105,
22994 domain.domain_id.0,
22995 "category",
22996 KeyRole::None,
22997 Some(VocabularyId(94_205)),
22998 ),
22999 row_role: None,
23000 vocabulary: Some(test_vocabulary(
23001 94_205,
23002 domain.domain_id.0,
23003 "issue04_category_codes",
23004 )),
23005 vocabulary_items: vec![test_vocabulary_item(94_305, 94_205, 1, "A")],
23006 }],
23007 },
23008 ))
23009 .expect_err("injected vocabulary failure should fail");
23010 assert!(
23011 error
23012 .to_string()
23013 .contains("issue04 injected vocabulary failure"),
23014 "{error}"
23015 );
23016
23017 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
23018 study_id: study.study_id,
23019 include_versions: false,
23020 }))
23021 .expect("dataset list should be readable");
23022 assert!(list.datasets.is_empty());
23023 assert!(!dataset_table_exists(
23024 &session,
23025 "study_issue04_failed_vocabulary_study",
23026 "issue04_failed_vocabulary_dataset_1_0_0"
23027 ));
23028
23029 let _ = fs::remove_dir_all(&lake_root);
23030 }
23031
23032 #[test]
23033 fn failed_output_provenance_does_not_create_complete_latest() {
23034 let repositories = AssetWorkflowRepositories::new();
23035 let service = repositories.service();
23036 let study = test_study("issue04_failed_provenance_study");
23037 repositories.seed_study(study.clone());
23038 let source_path = workspace_file("data/ingest.csv");
23039 let lake_root = temp_lake_root_path();
23040 let mut session = test_lake_session(&lake_root);
23041
23042 repositories.fail_next_transformation_outputs("issue04 injected provenance failure");
23043 let error = block_on(service.file_to_dataset(
23044 &mut session,
23045 FileToDatasetRequest {
23046 classification: ahri_tre_types::IngestClassification::derived_high(),
23047 study_id: study.study_id,
23048 dataset_asset_id: None,
23049 dataset_name: nc_name("issue04_failed_provenance_dataset"),
23050 dataset_version_id: None,
23051 input_path: source_path.display().to_string(),
23052 input_format: Some("csv".to_string()),
23053 header: None,
23054 delimiter: None,
23055 null_strings: Vec::new(),
23056 json_format: None,
23057 sheet: None,
23058 description: Some("Dataset with failed provenance".to_string()),
23059 agent_instructions: None,
23060 version_note: Some("Failed provenance materialization".to_string()),
23061 transformation: test_transformation(
23062 408,
23063 TransformationType::Ingest,
23064 "issue04 failed provenance materialization",
23065 ),
23066 variable_registrations: Vec::new(),
23067 },
23068 ))
23069 .expect_err("injected provenance failure should fail");
23070 assert!(
23071 error
23072 .to_string()
23073 .contains("issue04 injected provenance failure"),
23074 "{error}"
23075 );
23076
23077 let list = block_on(service.list_study_datasets(ListStudyDatasetsRequest {
23078 study_id: study.study_id,
23079 include_versions: false,
23080 }))
23081 .expect("dataset list should be readable");
23082 assert!(list.datasets.is_empty());
23083 assert!(
23084 repositories
23085 .transformations
23086 .state
23087 .outputs
23088 .lock()
23089 .expect("transformation output lock should not be poisoned")
23090 .is_empty()
23091 );
23092 assert!(!dataset_table_exists(
23093 &session,
23094 "study_issue04_failed_provenance_study",
23095 "issue04_failed_provenance_dataset_1_0_0"
23096 ));
23097
23098 let _ = fs::remove_dir_all(&lake_root);
23099 }
23100
23101 #[test]
23102 fn failed_rematerialization_preserves_previous_complete_latest_and_drops_new_table() {
23103 let repositories = AssetWorkflowRepositories::new();
23104 let service = repositories.service();
23105 let study = test_study("issue04_previous_latest_study");
23106 repositories.seed_study(study.clone());
23107 let source_path = workspace_file("data/ingest.csv");
23108 let lake_root = temp_lake_root_path();
23109 let mut session = test_lake_session(&lake_root);
23110
23111 let initial = block_on(service.file_to_dataset(
23112 &mut session,
23113 FileToDatasetRequest {
23114 classification: ahri_tre_types::IngestClassification::derived_high(),
23115 study_id: study.study_id,
23116 dataset_asset_id: None,
23117 dataset_name: nc_name("issue04_previous_latest_dataset"),
23118 dataset_version_id: None,
23119 input_path: source_path.display().to_string(),
23120 input_format: Some("csv".to_string()),
23121 header: None,
23122 delimiter: None,
23123 null_strings: Vec::new(),
23124 json_format: None,
23125 sheet: None,
23126 description: Some("Initial complete dataset".to_string()),
23127 agent_instructions: None,
23128 version_note: Some("Initial complete materialization".to_string()),
23129 transformation: test_transformation(
23130 405,
23131 TransformationType::Ingest,
23132 "issue04 initial file materialization",
23133 ),
23134 variable_registrations: Vec::new(),
23135 },
23136 ))
23137 .expect("initial materialization should succeed");
23138 let initial_version = initial
23139 .catalog
23140 .latest_version
23141 .clone()
23142 .expect("initial materialization should be latest");
23143
23144 repositories.fail_next_save_dataset("issue04 injected rematerialization failure");
23145 let error = block_on(service.file_to_dataset(
23146 &mut session,
23147 FileToDatasetRequest {
23148 classification: ahri_tre_types::IngestClassification::derived_high(),
23149 study_id: study.study_id,
23150 dataset_asset_id: Some(initial.catalog.asset.asset_id),
23151 dataset_name: initial.catalog.asset.name.clone(),
23152 dataset_version_id: None,
23153 input_path: source_path.display().to_string(),
23154 input_format: Some("csv".to_string()),
23155 header: None,
23156 delimiter: None,
23157 null_strings: Vec::new(),
23158 json_format: None,
23159 sheet: None,
23160 description: Some("Failed refresh dataset".to_string()),
23161 agent_instructions: None,
23162 version_note: Some("Failed refresh materialization #attempt".to_string()),
23163 transformation: test_transformation(
23164 406,
23165 TransformationType::Ingest,
23166 "issue04 failed refresh file materialization",
23167 ),
23168 variable_registrations: Vec::new(),
23169 },
23170 ))
23171 .expect_err("injected refresh metadata failure should fail");
23172 assert!(
23173 error
23174 .to_string()
23175 .contains("issue04 injected rematerialization failure"),
23176 "{error}"
23177 );
23178
23179 let metadata = block_on(service.get_study_dataset(GetStudyDatasetRequest {
23180 study_id: study.study_id,
23181 dataset_name: initial.catalog.asset.name.clone(),
23182 with_variables: true,
23183 }))
23184 .expect("dataset metadata should resolve")
23185 .expect("dataset should exist");
23186 assert_eq!(metadata.dataset, initial.dataset);
23187 assert_eq!(metadata.catalog.latest_version, Some(initial_version));
23188 assert_eq!(metadata.catalog.versions.len(), 1);
23189 assert_eq!(
23190 dataset_row_count(&session, &initial.lake_schema, &initial.lake_table),
23191 initial.row_count as i64
23192 );
23193 assert!(!dataset_table_exists(
23194 &session,
23195 &initial.lake_schema,
23196 "issue04_previous_latest_dataset_1_1_0"
23197 ));
23198 assert_eq!(
23199 repositories
23200 .transformations
23201 .state
23202 .outputs
23203 .lock()
23204 .expect("transformation output lock should not be poisoned")
23205 .len(),
23206 1
23207 );
23208
23209 let _ = fs::remove_dir_all(&lake_root);
23210 }
23211
23212 #[test]
23213 fn domain_metadata_workflows_register_and_enrich_variable_vocabulary() {
23214 let repositories = AssetWorkflowRepositories::new();
23215 let service = repositories.service();
23216 let domain = test_domain(90_101, "p2_2_metadata_domain");
23217 repositories.seed_domain(domain.clone());
23218
23219 let vocabulary = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23220 domain_id: domain.domain_id,
23221 vocabulary_id: None,
23222 name: nc_name("diagnosis_codes"),
23223 description: Some("Diagnosis vocabulary".to_string()),
23224 items: vec![
23225 RegisterVocabularyItemRequest {
23226 vocabulary_item_id: None,
23227 value: 1,
23228 code: "confirmed".to_string(),
23229 description: Some("Confirmed diagnosis".to_string()),
23230 },
23231 RegisterVocabularyItemRequest {
23232 vocabulary_item_id: None,
23233 value: 2,
23234 code: "probable".to_string(),
23235 description: Some("Probable diagnosis".to_string()),
23236 },
23237 ],
23238 force_metadata_updates: false,
23239 }))
23240 .expect("vocabulary should register");
23241 assert_eq!(vocabulary.vocabulary.name.as_str(), "diagnosis_codes");
23242 assert_eq!(vocabulary.vocabulary.vocabulary_id, VocabularyId(1));
23243 assert_eq!(vocabulary.items.len(), 2);
23244 assert_eq!(vocabulary.items[0].vocabulary_item_id, VocabularyItemId(1));
23245 assert_eq!(vocabulary.items[1].vocabulary_item_id, VocabularyItemId(2));
23246
23247 let variable = block_on(service.register_variable(RegisterVariableRequest {
23248 domain_id: domain.domain_id,
23249 variable_id: None,
23250 name: nc_name("diagnosis"),
23251 value_type_id: None,
23252 value_type: Some("enumeration".to_string()),
23253 value_format: None,
23254 vocabulary_id: Some(vocabulary.vocabulary.vocabulary_id),
23255 key_role: KeyRole::None,
23256 description: Some("Diagnosis status".to_string()),
23257 note: None,
23258 ontology_namespace: Some("https://example.test/ontology".to_string()),
23259 ontology_class: Some("DiagnosisStatus".to_string()),
23260 force_metadata_updates: false,
23261 }))
23262 .expect("variable should register by value type name");
23263 assert_eq!(variable.variable.name.as_str(), "diagnosis");
23264 assert_eq!(variable.variable.variable_id, VariableId(1));
23265 assert_eq!(variable.value_type.value_type, "enumeration");
23266 assert_eq!(variable.vocabulary_items.len(), 2);
23267
23268 let mismatch = block_on(service.register_variable(RegisterVariableRequest {
23269 domain_id: domain.domain_id,
23270 variable_id: None,
23271 name: nc_name("bad_value_type"),
23272 value_type_id: Some(ValueTypeId(1)),
23273 value_type: Some("xsd:string".to_string()),
23274 value_format: None,
23275 vocabulary_id: None,
23276 key_role: KeyRole::None,
23277 description: None,
23278 note: None,
23279 ontology_namespace: None,
23280 ontology_class: None,
23281 force_metadata_updates: false,
23282 }))
23283 .expect_err("conflicting value type id and name should fail");
23284 assert!(mismatch.to_string().contains("value_type_id 1"));
23285
23286 let enriched = block_on(service.register_variable(RegisterVariableRequest {
23287 domain_id: domain.domain_id,
23288 variable_id: None,
23289 name: nc_name("diagnosis"),
23290 value_type_id: Some(ValueTypeId(7)),
23291 value_type: Some("enumeration".to_string()),
23292 value_format: None,
23293 vocabulary_id: Some(vocabulary.vocabulary.vocabulary_id),
23294 key_role: KeyRole::None,
23295 description: Some("Curated diagnosis status".to_string()),
23296 note: Some("Reviewed after ingest".to_string()),
23297 ontology_namespace: Some("https://example.test/ontology".to_string()),
23298 ontology_class: Some("CuratedDiagnosisStatus".to_string()),
23299 force_metadata_updates: true,
23300 }))
23301 .expect("variable should enrich with force");
23302 assert_eq!(enriched.variable.variable_id, variable.variable.variable_id);
23303 assert_eq!(
23304 enriched.variable.description.as_deref(),
23305 Some("Curated diagnosis status")
23306 );
23307 assert_eq!(
23308 enriched.variable.ontology_class.as_deref(),
23309 Some("CuratedDiagnosisStatus")
23310 );
23311 }
23312
23313 #[test]
23314 fn semantic_catalog_facades_create_and_update_vocabulary_backed_variables() {
23315 let repositories = AssetWorkflowRepositories::new();
23316 let service = repositories.service();
23317 let study = test_study("cli_semantic_study");
23318 let domain = test_domain(90_151, "cli_semantic_domain");
23319 repositories.seed_study(study.clone());
23320 repositories.seed_study_domain(study.study_id, domain);
23321
23322 let vocabulary = block_on(service.add_vocabulary(AddVocabularyRequest {
23323 domain: "cli_semantic_domain".into(),
23324 name: "visit_status_codes".to_string(),
23325 description: Some("Visit status vocabulary".to_string()),
23326 items: "1:complete:Complete visit,2:missed:Missed visit".to_string(),
23327 }))
23328 .expect("vocabulary command facade should create vocabulary");
23329 assert_eq!(vocabulary.vocabulary.name.as_str(), "visit_status_codes");
23330 assert_eq!(vocabulary.items.len(), 2);
23331
23332 let variable = block_on(service.add_variable(AddVariableRequest {
23333 domain: "cli_semantic_domain".into(),
23334 name: "visit_status".to_string(),
23335 value_type: "enumeration".to_string(),
23336 value_format: None,
23337 key_role: Some("none".to_string()),
23338 description: Some("Visit completion status".to_string()),
23339 ontology_namespace: None,
23340 ontology_class: None,
23341 vocabulary: Some("visit_status_codes".to_string()),
23342 vocabulary_items: Some("complete,missed".to_string()),
23343 }))
23344 .expect("categorical variable should validate governed vocabulary items");
23345 assert_eq!(variable.variable.name.as_str(), "visit_status");
23346 assert_eq!(variable.value_type.value_type, "enumeration");
23347 assert_eq!(
23348 variable
23349 .vocabulary
23350 .as_ref()
23351 .map(|value| value.name.as_str()),
23352 Some("visit_status_codes")
23353 );
23354
23355 let updated = block_on(service.update_variable(UpdateVariableRequest {
23356 variable: VariableSelector::Name {
23357 domain: "cli_semantic_domain".into(),
23358 name: "visit_status".to_string(),
23359 },
23360 value_type: None,
23361 value_format: Some("integer-coded".to_string()),
23362 key_role: None,
23363 description: Some("Curated visit completion status".to_string()),
23364 ontology_namespace: Some("https://example.test/visit".to_string()),
23365 ontology_class: Some("VisitStatus".to_string()),
23366 vocabulary: None,
23367 }))
23368 .expect("variable update should preserve vocabulary and update metadata");
23369 assert_eq!(
23370 updated.variable.description.as_deref(),
23371 Some("Curated visit completion status")
23372 );
23373 assert_eq!(
23374 updated.variable.value_format.as_deref(),
23375 Some("integer-coded")
23376 );
23377 assert_eq!(updated.vocabulary_items.len(), 2);
23378
23379 let listed = block_on(service.list_variables(ListVariablesRequest {
23380 domain: Some("cli_semantic_domain".into()),
23381 study: None,
23382 }))
23383 .expect("variables should list by domain");
23384 assert_eq!(listed.len(), 1);
23385 assert_eq!(listed[0].variable.name.as_str(), "visit_status");
23386
23387 let vocabularies_by_domain = block_on(service.list_vocabularies(ListVocabulariesRequest {
23388 domain: Some("cli_semantic_domain".into()),
23389 study: None,
23390 }))
23391 .expect("vocabularies should list by domain");
23392 assert_eq!(vocabularies_by_domain.len(), 1);
23393 assert_eq!(
23394 vocabularies_by_domain[0].vocabulary.name.as_str(),
23395 "visit_status_codes"
23396 );
23397
23398 let vocabularies_by_study = block_on(service.list_vocabularies(ListVocabulariesRequest {
23399 domain: None,
23400 study: Some("cli_semantic_study".into()),
23401 }))
23402 .expect("vocabularies should list by study");
23403 assert_eq!(vocabularies_by_study, vocabularies_by_domain);
23404 }
23405
23406 #[test]
23407 fn dictionary_facades_accept_id_selectors_for_variable_vocabulary_and_domain_filters() {
23408 let repositories = AssetWorkflowRepositories::new();
23409 let service = repositories.service();
23410 let domain = test_domain(90_153, "dictionary_id_domain");
23411 repositories.seed_domain(domain.clone());
23412
23413 let vocabulary = block_on(service.add_vocabulary(AddVocabularyRequest {
23414 domain: DomainSelector::Id {
23415 domain_id: domain.domain_id,
23416 },
23417 name: "result_codes".to_string(),
23418 description: Some("Result vocabulary".to_string()),
23419 items: "1:positive:Positive,2:negative:Negative".to_string(),
23420 }))
23421 .expect("vocabulary should create with domain id selector");
23422
23423 let variable = block_on(service.add_variable(AddVariableRequest {
23424 domain: DomainSelector::Id {
23425 domain_id: domain.domain_id,
23426 },
23427 name: "result".to_string(),
23428 value_type: "enumeration".to_string(),
23429 value_format: None,
23430 key_role: Some("none".to_string()),
23431 description: None,
23432 ontology_namespace: None,
23433 ontology_class: None,
23434 vocabulary: Some("result_codes".to_string()),
23435 vocabulary_items: Some("positive,negative".to_string()),
23436 }))
23437 .expect("variable should create with domain id selector");
23438
23439 let variable_by_id = block_on(service.get_variable(GetVariableRequest {
23440 variable: VariableSelector::Id {
23441 variable_id: variable.variable.variable_id,
23442 },
23443 }))
23444 .expect("variable id lookup should not fail")
23445 .expect("variable id should resolve");
23446 assert_eq!(variable_by_id.variable.name.as_str(), "result");
23447
23448 let vocabulary_by_id = block_on(service.get_vocabulary(GetVocabularyRequest {
23449 vocabulary: VocabularySelector::Id {
23450 vocabulary_id: vocabulary.vocabulary.vocabulary_id,
23451 },
23452 }))
23453 .expect("vocabulary id lookup should not fail")
23454 .expect("vocabulary id should resolve");
23455 assert_eq!(vocabulary_by_id.vocabulary.name.as_str(), "result_codes");
23456
23457 let updated = block_on(service.update_variable(UpdateVariableRequest {
23458 variable: VariableSelector::Id {
23459 variable_id: variable.variable.variable_id,
23460 },
23461 value_type: None,
23462 value_format: Some("integer-coded".to_string()),
23463 key_role: None,
23464 description: Some("Curated result".to_string()),
23465 ontology_namespace: None,
23466 ontology_class: None,
23467 vocabulary: None,
23468 }))
23469 .expect("variable id update should resolve existing variable");
23470 assert_eq!(
23471 updated.variable.description.as_deref(),
23472 Some("Curated result")
23473 );
23474 assert_eq!(
23475 updated.variable.value_format.as_deref(),
23476 Some("integer-coded")
23477 );
23478
23479 let variables_by_domain_id = block_on(service.list_variables(ListVariablesRequest {
23480 domain: Some(DomainSelector::Id {
23481 domain_id: domain.domain_id,
23482 }),
23483 study: None,
23484 }))
23485 .expect("variables should list by domain id");
23486 assert_eq!(variables_by_domain_id.len(), 1);
23487 assert_eq!(
23488 variables_by_domain_id[0].variable.variable_id,
23489 variable.variable.variable_id
23490 );
23491
23492 let vocabularies_by_domain_id =
23493 block_on(service.list_vocabularies(ListVocabulariesRequest {
23494 domain: Some(DomainSelector::Id {
23495 domain_id: domain.domain_id,
23496 }),
23497 study: None,
23498 }))
23499 .expect("vocabularies should list by domain id");
23500 assert_eq!(vocabularies_by_domain_id.len(), 1);
23501 assert_eq!(
23502 vocabularies_by_domain_id[0].vocabulary.vocabulary_id,
23503 vocabulary.vocabulary.vocabulary_id
23504 );
23505
23506 let missing = block_on(service.get_variable(GetVariableRequest {
23507 variable: VariableSelector::Id {
23508 variable_id: VariableId(99_999),
23509 },
23510 }))
23511 .expect("missing variable id lookup should not fail");
23512 assert!(missing.is_none());
23513 }
23514
23515 #[test]
23516 fn semantic_catalog_facades_report_missing_references_and_bad_categorical_items() {
23517 let repositories = AssetWorkflowRepositories::new();
23518 let service = repositories.service();
23519 repositories.seed_domain(test_domain(90_152, "cli_validation_domain"));
23520
23521 let missing_domain = block_on(service.add_vocabulary(AddVocabularyRequest {
23522 domain: "missing_domain".into(),
23523 name: "codes".to_string(),
23524 description: None,
23525 items: "1:yes".to_string(),
23526 }))
23527 .expect_err("missing vocabulary domain should fail");
23528 assert!(missing_domain.to_string().contains("domain not found"));
23529
23530 let missing_vocabulary = block_on(service.add_variable(AddVariableRequest {
23531 domain: "cli_validation_domain".into(),
23532 name: "status".to_string(),
23533 value_type: "enumeration".to_string(),
23534 value_format: None,
23535 key_role: None,
23536 description: None,
23537 ontology_namespace: None,
23538 ontology_class: None,
23539 vocabulary: Some("missing_codes".to_string()),
23540 vocabulary_items: None,
23541 }))
23542 .expect_err("missing variable vocabulary should fail");
23543 assert!(
23544 missing_vocabulary
23545 .to_string()
23546 .contains("vocabulary missing_codes")
23547 );
23548
23549 block_on(service.add_vocabulary(AddVocabularyRequest {
23550 domain: "cli_validation_domain".into(),
23551 name: "status_codes".to_string(),
23552 description: None,
23553 items: "1:yes,2:no".to_string(),
23554 }))
23555 .expect("fixture vocabulary should create");
23556 let missing_item = block_on(service.add_variable(AddVariableRequest {
23557 domain: "cli_validation_domain".into(),
23558 name: "bad_status".to_string(),
23559 value_type: "enumeration".to_string(),
23560 value_format: None,
23561 key_role: None,
23562 description: None,
23563 ontology_namespace: None,
23564 ontology_class: None,
23565 vocabulary: Some("status_codes".to_string()),
23566 vocabulary_items: Some("maybe".to_string()),
23567 }))
23568 .expect_err("unknown categorical item should fail");
23569 assert!(missing_item.to_string().contains("vocabulary item maybe"));
23570 }
23571
23572 #[test]
23573 fn vocabulary_mapping_workflows_register_and_summarize_cross_domain_mappings() {
23574 let repositories = AssetWorkflowRepositories::new();
23575 let service = repositories.service();
23576 let source_domain = test_domain(90_141, "p2_4_source_domain");
23577 let target_domain = test_domain(90_142, "p2_4_target_domain");
23578 repositories.seed_domain(source_domain.clone());
23579 repositories.seed_domain(target_domain.clone());
23580
23581 let source = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23582 domain_id: source_domain.domain_id,
23583 vocabulary_id: Some(VocabularyId(91_410)),
23584 name: nc_name("source_status"),
23585 description: None,
23586 items: vec![
23587 RegisterVocabularyItemRequest {
23588 vocabulary_item_id: Some(VocabularyItemId(91_411)),
23589 value: 1,
23590 code: "yes".to_string(),
23591 description: Some("Yes".to_string()),
23592 },
23593 RegisterVocabularyItemRequest {
23594 vocabulary_item_id: Some(VocabularyItemId(91_412)),
23595 value: 2,
23596 code: "maybe".to_string(),
23597 description: Some("Maybe".to_string()),
23598 },
23599 ],
23600 force_metadata_updates: false,
23601 }))
23602 .expect("source vocabulary should register");
23603 assert_eq!(source.vocabulary.vocabulary_id, VocabularyId(91_410));
23604 assert_eq!(source.items[0].vocabulary_item_id, VocabularyItemId(91_411));
23605 let target = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23606 domain_id: target_domain.domain_id,
23607 vocabulary_id: Some(VocabularyId(91_420)),
23608 name: nc_name("target_status"),
23609 description: None,
23610 items: vec![
23611 RegisterVocabularyItemRequest {
23612 vocabulary_item_id: Some(VocabularyItemId(91_421)),
23613 value: 10,
23614 code: "positive".to_string(),
23615 description: Some("Positive".to_string()),
23616 },
23617 RegisterVocabularyItemRequest {
23618 vocabulary_item_id: Some(VocabularyItemId(91_422)),
23619 value: 20,
23620 code: "uncertain".to_string(),
23621 description: Some("Uncertain".to_string()),
23622 },
23623 ],
23624 force_metadata_updates: false,
23625 }))
23626 .expect("target vocabulary should register");
23627
23628 let first = block_on(service.register_vocabulary_mapping(
23629 RegisterVocabularyMappingRequest {
23630 from: VocabularyItemReference {
23631 vocabulary_item_id: None,
23632 domain_id: Some(source_domain.domain_id),
23633 vocabulary_id: None,
23634 vocabulary_name: Some(source.vocabulary.name.clone()),
23635 value: None,
23636 code: Some("yes".to_string()),
23637 },
23638 to: VocabularyItemReference {
23639 vocabulary_item_id: Some(target.items[0].vocabulary_item_id),
23640 domain_id: Some(target_domain.domain_id),
23641 vocabulary_id: Some(target.vocabulary.vocabulary_id),
23642 vocabulary_name: None,
23643 value: Some(10),
23644 code: None,
23645 },
23646 },
23647 ))
23648 .expect("cross-domain mapping should register");
23649 assert_eq!(first.mapping.vocabulary_mapping_id, 1);
23650 assert_eq!(first.from_vocabulary.domain_id, source_domain.domain_id);
23651 assert_eq!(first.to_vocabulary.domain_id, target_domain.domain_id);
23652
23653 let duplicate = block_on(service.register_vocabulary_mapping(
23654 RegisterVocabularyMappingRequest {
23655 from: VocabularyItemReference {
23656 vocabulary_item_id: Some(source.items[0].vocabulary_item_id),
23657 domain_id: None,
23658 vocabulary_id: None,
23659 vocabulary_name: None,
23660 value: None,
23661 code: None,
23662 },
23663 to: VocabularyItemReference {
23664 vocabulary_item_id: Some(target.items[0].vocabulary_item_id),
23665 domain_id: None,
23666 vocabulary_id: None,
23667 vocabulary_name: None,
23668 value: None,
23669 code: None,
23670 },
23671 },
23672 ))
23673 .expect("duplicate mapping should be idempotent");
23674 assert_eq!(duplicate.mapping, first.mapping);
23675
23676 block_on(
23677 service.register_vocabulary_mapping(RegisterVocabularyMappingRequest {
23678 from: VocabularyItemReference {
23679 vocabulary_item_id: Some(source.items[1].vocabulary_item_id),
23680 domain_id: None,
23681 vocabulary_id: None,
23682 vocabulary_name: None,
23683 value: None,
23684 code: None,
23685 },
23686 to: VocabularyItemReference {
23687 vocabulary_item_id: Some(target.items[1].vocabulary_item_id),
23688 domain_id: None,
23689 vocabulary_id: None,
23690 vocabulary_name: None,
23691 value: None,
23692 code: None,
23693 },
23694 }),
23695 )
23696 .expect("many mapping should register");
23697 block_on(
23698 service.register_vocabulary_mapping(RegisterVocabularyMappingRequest {
23699 from: VocabularyItemReference {
23700 vocabulary_item_id: Some(source.items[0].vocabulary_item_id),
23701 domain_id: None,
23702 vocabulary_id: None,
23703 vocabulary_name: None,
23704 value: None,
23705 code: None,
23706 },
23707 to: VocabularyItemReference {
23708 vocabulary_item_id: Some(target.items[1].vocabulary_item_id),
23709 domain_id: None,
23710 vocabulary_id: None,
23711 vocabulary_name: None,
23712 value: None,
23713 code: None,
23714 },
23715 }),
23716 )
23717 .expect("one-to-many mapping should register");
23718
23719 let source_to_target = block_on(service.list_vocabulary_mappings(
23720 ListVocabularyMappingsRequest {
23721 from_vocabulary_id: Some(source.vocabulary.vocabulary_id),
23722 to_vocabulary_id: Some(target.vocabulary.vocabulary_id),
23723 involving_vocabulary_id: None,
23724 item_id: None,
23725 },
23726 ))
23727 .expect("filtered mappings should list");
23728 assert_eq!(source_to_target.len(), 3);
23729
23730 let summary = block_on(service.vocabulary_mapping_summary(
23731 VocabularyMappingSummaryRequest {
23732 vocabulary_id: source.vocabulary.vocabulary_id,
23733 },
23734 ))
23735 .expect("summary should read");
23736 assert!(summary.has_mappings);
23737 assert_eq!(summary.outgoing_count, 3);
23738 assert_eq!(summary.incoming_count, 0);
23739 assert_eq!(summary.mapped_vocabularies.len(), 1);
23740 assert_eq!(
23741 summary.mapped_vocabularies[0].vocabulary.vocabulary_id,
23742 target.vocabulary.vocabulary_id
23743 );
23744 }
23745
23746 #[test]
23747 fn vocabulary_mapping_rejects_invalid_reference_context() {
23748 let repositories = AssetWorkflowRepositories::new();
23749 let service = repositories.service();
23750 let domain = test_domain(90_143, "p2_4_invalid_mapping_domain");
23751 repositories.seed_domain(domain.clone());
23752 let vocabulary = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23753 domain_id: domain.domain_id,
23754 vocabulary_id: Some(VocabularyId(91_430)),
23755 name: nc_name("mapping_references"),
23756 description: None,
23757 items: vec![RegisterVocabularyItemRequest {
23758 vocabulary_item_id: Some(VocabularyItemId(91_431)),
23759 value: 1,
23760 code: "one".to_string(),
23761 description: None,
23762 }],
23763 force_metadata_updates: false,
23764 }))
23765 .expect("vocabulary should register");
23766
23767 let error = block_on(service.register_vocabulary_mapping(
23768 RegisterVocabularyMappingRequest {
23769 from: VocabularyItemReference {
23770 vocabulary_item_id: Some(vocabulary.items[0].vocabulary_item_id),
23771 domain_id: None,
23772 vocabulary_id: None,
23773 vocabulary_name: None,
23774 value: Some(2),
23775 code: None,
23776 },
23777 to: VocabularyItemReference {
23778 vocabulary_item_id: Some(vocabulary.items[0].vocabulary_item_id),
23779 domain_id: None,
23780 vocabulary_id: None,
23781 vocabulary_name: None,
23782 value: None,
23783 code: None,
23784 },
23785 },
23786 ))
23787 .expect_err("conflicting item context should fail");
23788 assert!(error.to_string().contains("has value 1, not 2"));
23789 }
23790
23791 #[test]
23792 fn vocabulary_items_are_unique_by_value_and_code_within_vocabulary() {
23793 let repositories = AssetWorkflowRepositories::new();
23794 let service = repositories.service();
23795 let domain = test_domain(90_144, "p2_4_unique_items_domain");
23796 repositories.seed_domain(domain.clone());
23797
23798 block_on(service.register_vocabulary(RegisterVocabularyRequest {
23799 domain_id: domain.domain_id,
23800 vocabulary_id: Some(VocabularyId(91_440)),
23801 name: nc_name("unique_items"),
23802 description: None,
23803 items: vec![RegisterVocabularyItemRequest {
23804 vocabulary_item_id: Some(VocabularyItemId(91_441)),
23805 value: 1,
23806 code: "one".to_string(),
23807 description: None,
23808 }],
23809 force_metadata_updates: false,
23810 }))
23811 .expect("initial vocabulary item should register");
23812
23813 let duplicate_value = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23814 domain_id: domain.domain_id,
23815 vocabulary_id: Some(VocabularyId(91_440)),
23816 name: nc_name("unique_items"),
23817 description: None,
23818 items: vec![RegisterVocabularyItemRequest {
23819 vocabulary_item_id: Some(VocabularyItemId(91_442)),
23820 value: 1,
23821 code: "uno".to_string(),
23822 description: None,
23823 }],
23824 force_metadata_updates: false,
23825 }))
23826 .expect_err("duplicate item value should fail");
23827 assert!(duplicate_value.to_string().contains("vocabulary item"));
23828
23829 let duplicate_code = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23830 domain_id: domain.domain_id,
23831 vocabulary_id: Some(VocabularyId(91_440)),
23832 name: nc_name("unique_items"),
23833 description: None,
23834 items: vec![RegisterVocabularyItemRequest {
23835 vocabulary_item_id: Some(VocabularyItemId(91_443)),
23836 value: 2,
23837 code: "one".to_string(),
23838 description: None,
23839 }],
23840 force_metadata_updates: false,
23841 }))
23842 .expect_err("duplicate item code should fail");
23843 assert!(duplicate_code.to_string().contains("vocabulary item"));
23844 }
23845
23846 #[test]
23847 fn register_vocabulary_deduplicates_identical_items_within_single_request() {
23848 let repositories = AssetWorkflowRepositories::new();
23849 let service = repositories.service();
23850 let domain = test_domain(90_145, "p2_4_duplicate_batch_domain");
23851 repositories.seed_domain(domain.clone());
23852
23853 let vocabulary = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23854 domain_id: domain.domain_id,
23855 vocabulary_id: Some(VocabularyId(91_450)),
23856 name: nc_name("duplicate_batch_items"),
23857 description: None,
23858 items: vec![
23859 RegisterVocabularyItemRequest {
23860 vocabulary_item_id: None,
23861 value: 1,
23862 code: "one".to_string(),
23863 description: Some("One".to_string()),
23864 },
23865 RegisterVocabularyItemRequest {
23866 vocabulary_item_id: None,
23867 value: 1,
23868 code: "one".to_string(),
23869 description: Some("One".to_string()),
23870 },
23871 ],
23872 force_metadata_updates: false,
23873 }))
23874 .expect("duplicate identical items should be treated as one item");
23875
23876 assert_eq!(vocabulary.items.len(), 1);
23877 assert_eq!(vocabulary.items[0].value, 1);
23878 assert_eq!(vocabulary.items[0].code, "one");
23879 assert_eq!(vocabulary.items[0].description.as_deref(), Some("One"));
23880 }
23881
23882 #[test]
23883 fn dataset_variable_metadata_reads_records_and_locks_linked_type_changes() {
23884 let repositories = AssetWorkflowRepositories::new();
23885 let service = repositories.service();
23886 let domain = test_domain(90_102, "p2_2_linked_metadata_domain");
23887 repositories.seed_domain(domain.clone());
23888 let dataset = DatasetRecord {
23889 dataset_id: VersionId(uuid::Uuid::new_v4()),
23890 };
23891 repositories.seed_dataset(dataset.clone());
23892
23893 let variable = block_on(service.register_variable(RegisterVariableRequest {
23894 domain_id: domain.domain_id,
23895 variable_id: Some(VariableId(91_001)),
23896 name: nc_name("participant_id"),
23897 value_type_id: None,
23898 value_type: Some("xsd:integer".to_string()),
23899 value_format: None,
23900 vocabulary_id: None,
23901 key_role: KeyRole::Record,
23902 description: Some("Participant identifier".to_string()),
23903 note: None,
23904 ontology_namespace: None,
23905 ontology_class: None,
23906 force_metadata_updates: false,
23907 }))
23908 .expect("variable should register");
23909
23910 let linked = block_on(service.register_dataset_variables(
23911 RegisterDatasetVariablesRequest {
23912 dataset_id: dataset.dataset_id,
23913 domain_id: domain.domain_id,
23914 variables: vec![DatasetVariableRegistrationRequest {
23915 variable: variable.variable.clone(),
23916 row_role: Some(KeyRole::Record),
23917 vocabulary: None,
23918 vocabulary_items: Vec::new(),
23919 }],
23920 },
23921 ))
23922 .expect("dataset variable should link");
23923 assert_eq!(linked[0].link.row_role, KeyRole::Record);
23924
23925 let metadata = block_on(service.get_dataset_variables(GetDatasetVariablesRequest {
23926 dataset_id: dataset.dataset_id,
23927 }))
23928 .expect("dataset variable metadata should read");
23929 assert_eq!(metadata.dataset, dataset);
23930 assert_eq!(metadata.variables.len(), 1);
23931 assert_eq!(
23932 metadata.variables[0].variable.name.as_str(),
23933 "participant_id"
23934 );
23935 assert_eq!(metadata.variables[0].value_type.value_type, "xsd:integer");
23936 assert_eq!(metadata.variables[0].link.row_role, KeyRole::Record);
23937
23938 let external_variable = block_on(service.register_variable(RegisterVariableRequest {
23939 domain_id: domain.domain_id,
23940 variable_id: Some(VariableId(91_002)),
23941 name: nc_name("source_participant_id"),
23942 value_type_id: None,
23943 value_type: Some("xsd:string".to_string()),
23944 value_format: None,
23945 vocabulary_id: None,
23946 key_role: KeyRole::External,
23947 description: Some("Source participant identifier".to_string()),
23948 note: None,
23949 ontology_namespace: None,
23950 ontology_class: None,
23951 force_metadata_updates: false,
23952 }))
23953 .expect("external variable should register");
23954
23955 let external_link = block_on(service.register_dataset_variables(
23956 RegisterDatasetVariablesRequest {
23957 dataset_id: dataset.dataset_id,
23958 domain_id: domain.domain_id,
23959 variables: vec![DatasetVariableRegistrationRequest {
23960 variable: external_variable.variable.clone(),
23961 row_role: Some(KeyRole::External),
23962 vocabulary: None,
23963 vocabulary_items: Vec::new(),
23964 }],
23965 },
23966 ))
23967 .expect("external dataset variable should link");
23968 assert_eq!(external_link[0].variable.key_role, KeyRole::External);
23969 assert_eq!(external_link[0].link.row_role, KeyRole::External);
23970
23971 let vocabulary = block_on(service.register_vocabulary(RegisterVocabularyRequest {
23972 domain_id: domain.domain_id,
23973 vocabulary_id: None,
23974 name: nc_name("participant_id_codes"),
23975 description: Some("Integer-coded participant categories".to_string()),
23976 items: vec![
23977 RegisterVocabularyItemRequest {
23978 vocabulary_item_id: None,
23979 value: 1,
23980 code: "first".to_string(),
23981 description: None,
23982 },
23983 RegisterVocabularyItemRequest {
23984 vocabulary_item_id: None,
23985 value: 2,
23986 code: "second".to_string(),
23987 description: None,
23988 },
23989 ],
23990 force_metadata_updates: false,
23991 }))
23992 .expect("vocabulary should register");
23993 let enumerated = block_on(service.register_variable(RegisterVariableRequest {
23994 domain_id: domain.domain_id,
23995 variable_id: None,
23996 name: nc_name("participant_id"),
23997 value_type_id: None,
23998 value_type: Some("enumeration".to_string()),
23999 value_format: None,
24000 vocabulary_id: Some(vocabulary.vocabulary.vocabulary_id),
24001 key_role: KeyRole::Record,
24002 description: Some("Participant identifier as governed integer category".to_string()),
24003 note: None,
24004 ontology_namespace: None,
24005 ontology_class: None,
24006 force_metadata_updates: true,
24007 }))
24008 .expect("linked integer variable should become enumeration without storage migration");
24009 assert_eq!(
24010 enumerated.variable.variable_id,
24011 variable.variable.variable_id
24012 );
24013 assert_eq!(enumerated.value_type.value_type, "enumeration");
24014 assert_eq!(
24015 enumerated
24016 .vocabulary
24017 .as_ref()
24018 .map(|vocabulary| vocabulary.name.as_str()),
24019 Some("participant_id_codes")
24020 );
24021
24022 let error = block_on(service.register_variable(RegisterVariableRequest {
24023 domain_id: domain.domain_id,
24024 variable_id: None,
24025 name: nc_name("participant_id"),
24026 value_type_id: None,
24027 value_type: Some("xsd:string".to_string()),
24028 value_format: None,
24029 vocabulary_id: None,
24030 key_role: KeyRole::Record,
24031 description: Some("Participant identifier as text".to_string()),
24032 note: None,
24033 ontology_namespace: None,
24034 ontology_class: None,
24035 force_metadata_updates: true,
24036 }))
24037 .expect_err("linked variable type change should fail");
24038 assert!(
24039 error
24040 .to_string()
24041 .contains("storage-aware migration workflow"),
24042 "{error}"
24043 );
24044 }
24045
24046 #[test]
24047 fn export_dataset_writes_conventional_filename_and_export_provenance() {
24048 let repositories = AssetWorkflowRepositories::new();
24049 let service = repositories.service();
24050 let study = test_study("p1_15_dataset_export_study");
24051 repositories.seed_study(study.clone());
24052 let source_path = workspace_file("data/ingest.csv");
24053 let lake_root = temp_lake_root_path();
24054 let mut session = test_lake_session(&lake_root);
24055 let materialized = block_on(service.file_to_dataset(
24056 &mut session,
24057 FileToDatasetRequest {
24058 classification: ahri_tre_types::IngestClassification::derived_high(),
24059 study_id: study.study_id,
24060 dataset_asset_id: None,
24061 dataset_name: nc_name("p1_15_export_dataset"),
24062 dataset_version_id: None,
24063 input_path: source_path.display().to_string(),
24064 input_format: Some("csv".to_string()),
24065 header: None,
24066 delimiter: None,
24067 null_strings: Vec::new(),
24068 json_format: None,
24069 sheet: None,
24070 description: Some("Dataset to export".to_string()),
24071 agent_instructions: None,
24072 version_note: Some("Initial export dataset".to_string()),
24073 transformation: test_transformation(
24074 190,
24075 TransformationType::Ingest,
24076 "ingest dataset before export",
24077 ),
24078 variable_registrations: vec![DatasetVariableRegistrationRequest {
24079 variable: test_variable(10_101, 90_101, "country", KeyRole::Record, None),
24080 row_role: Some(KeyRole::Record),
24081 vocabulary: None,
24082 vocabulary_items: Vec::new(),
24083 }],
24084 },
24085 ))
24086 .expect("dataset should materialize before export");
24087 let target_dir = lake_root.join("exports");
24088 let export_transformation =
24089 test_transformation(191, TransformationType::Export, "export dataset parquet");
24090
24091 let exported = block_on(service.export_dataset(
24092 &mut session,
24093 ExportDatasetRequest {
24094 dataset_id: materialized.dataset.dataset_id,
24095 target_dir: target_dir.clone(),
24096 format: DatasetExportFormat::Parquet,
24097 limit: None,
24098 compress: false,
24099 overwrite: false,
24100 transformation: export_transformation.clone(),
24101 },
24102 ))
24103 .expect("dataset export should succeed");
24104
24105 assert_eq!(
24106 exported.target_path,
24107 target_dir.join("p1_15_export_dataset_1_0_0.parquet")
24108 );
24109 assert!(exported.compressed);
24110 assert_eq!(exported.row_count, 10);
24111 assert_eq!(exported.column_count, 5);
24112 assert!(
24113 exported
24114 .warnings
24115 .iter()
24116 .any(|warning| warning.contains("Arrow field category has no registered")),
24117 "{:?}",
24118 exported.warnings
24119 );
24120 assert_eq!(
24121 fs::read(&exported.target_path)
24122 .expect("parquet export should read")
24123 .get(..4),
24124 Some(&b"PAR1"[..])
24125 );
24126 let exported_table = ahri_tre_tabular::read_parquet_file(&exported.target_path)
24127 .expect("parquet should read");
24128 assert_eq!(
24129 exported_table
24130 .schema()
24131 .metadata()
24132 .get(metadata_keys::ASSET_NAME)
24133 .map(String::as_str),
24134 Some("p1_15_export_dataset")
24135 );
24136 assert_eq!(
24137 exported_table
24138 .schema()
24139 .field_with_name("country")
24140 .expect("country field should exist")
24141 .metadata()
24142 .get(metadata_keys::DESCRIPTION)
24143 .map(String::as_str),
24144 Some("Variable country")
24145 );
24146 assert_eq!(
24147 exported_table
24148 .schema()
24149 .field_with_name("country")
24150 .expect("country field should exist")
24151 .metadata()
24152 .get(metadata_keys::ROW_ROLE)
24153 .map(String::as_str),
24154 Some("record")
24155 );
24156 assert_eq!(exported.lineage.transformation, export_transformation);
24157 assert_eq!(
24158 exported
24159 .lineage
24160 .inputs
24161 .iter()
24162 .map(|input| input.version_id)
24163 .collect::<Vec<_>>(),
24164 vec![materialized.dataset.dataset_id]
24165 );
24166 assert!(exported.lineage.outputs.is_empty());
24167
24168 let duplicate = block_on(service.export_dataset(
24169 &mut session,
24170 ExportDatasetRequest {
24171 dataset_id: materialized.dataset.dataset_id,
24172 target_dir,
24173 format: DatasetExportFormat::Parquet,
24174 limit: None,
24175 compress: false,
24176 overwrite: false,
24177 transformation: test_transformation(
24178 192,
24179 TransformationType::Export,
24180 "duplicate dataset export",
24181 ),
24182 },
24183 ))
24184 .expect_err("export should not overwrite by default");
24185 assert!(matches!(duplicate, AppError::Validation(_)));
24186
24187 let _ = fs::remove_dir_all(&lake_root);
24188 }
24189
24190 #[test]
24191 fn export_datafile_materializes_decrypted_payload_with_conventional_filename() {
24192 let repositories = AssetWorkflowRepositories::new();
24193 let service = repositories.service();
24194 let study = test_study("p1_15_datafile_export_study");
24195 repositories.seed_study(study.clone());
24196 let source_path = workspace_file("data/ingest.csv");
24197 let lake_root = temp_lake_root_path();
24198 let session = test_lake_session(&lake_root);
24199 let ingested = block_on(service.ingest_file(IngestFileRequest {
24200 classification: ahri_tre_types::IngestClassification::derived_high(),
24201 study_id: study.study_id,
24202 asset_id: None,
24203 asset_name: nc_name("p1_15_export_datafile"),
24204 version_id: None,
24205 source: IngestFileSource::LocalPath {
24206 path: source_path.display().to_string(),
24207 },
24208 lake_root: lake_root.display().to_string(),
24209 edam_format: None,
24210 compress: true,
24211 encrypt: None,
24212 description: Some("Datafile to export".to_string()),
24213 agent_instructions: None,
24214 version_note: Some("Initial export datafile".to_string()),
24215 transformation: test_transformation(
24216 193,
24217 TransformationType::Ingest,
24218 "ingest datafile before export",
24219 ),
24220 }))
24221 .expect("datafile should ingest before export");
24222 let target_dir = lake_root.join("datafile_exports");
24223
24224 let exported = block_on(service.export_datafile(
24225 &session,
24226 ExportDataFileRequest {
24227 datafile_id: ingested.datafile.datafile_id,
24228 target_dir: target_dir.clone(),
24229 compress: false,
24230 overwrite: false,
24231 transformation: test_transformation(
24232 194,
24233 TransformationType::Export,
24234 "export datafile",
24235 ),
24236 },
24237 ))
24238 .expect("datafile export should succeed");
24239
24240 assert_eq!(
24241 exported.target_path,
24242 target_dir.join("p1_15_export_datafile_1_0_0.csv")
24243 );
24244 assert!(!exported.compressed);
24245 assert_eq!(
24246 fs::read(&exported.target_path).expect("exported datafile should read"),
24247 fs::read(&source_path).expect("source datafile should read")
24248 );
24249 assert_eq!(
24250 exported
24251 .lineage
24252 .inputs
24253 .iter()
24254 .map(|input| input.version_id)
24255 .collect::<Vec<_>>(),
24256 vec![ingested.datafile.datafile_id]
24257 );
24258 assert!(exported.lineage.outputs.is_empty());
24259
24260 let compressed = block_on(service.export_datafile(
24261 &session,
24262 ExportDataFileRequest {
24263 datafile_id: ingested.datafile.datafile_id,
24264 target_dir,
24265 compress: true,
24266 overwrite: false,
24267 transformation: test_transformation(
24268 195,
24269 TransformationType::Export,
24270 "export compressed datafile",
24271 ),
24272 },
24273 ))
24274 .expect("compressed datafile export should succeed");
24275 assert_eq!(
24276 compressed
24277 .target_path
24278 .file_name()
24279 .and_then(|value| value.to_str()),
24280 Some("p1_15_export_datafile_1_0_0.csv.zst")
24281 );
24282 let decoded = zstd::stream::decode_all(
24283 fs::read(&compressed.target_path)
24284 .expect("compressed datafile export should read")
24285 .as_slice(),
24286 )
24287 .expect("compressed datafile export should decode");
24288 assert_eq!(decoded, fs::read(source_path).expect("source should read"));
24289
24290 let _ = fs::remove_dir_all(&lake_root);
24291 }
24292
24293 #[test]
24294 fn sql_to_dataset_loads_duckdb_source_with_inferred_variables_and_vocabularies() {
24295 let repositories = AssetWorkflowRepositories::new();
24296 let service = repositories.service();
24297 let study = test_study("p1_14_sql_dataset_study");
24298 let domain = test_domain(90_014, "p1_14_sql_domain");
24299 repositories.seed_study(study.clone());
24300 repositories.seed_study_domain(study.study_id, domain.clone());
24301 let lake_root = temp_lake_root_path();
24302 let source_path = lake_root.join("meta_test.duckdb");
24303 seed_meta_test_duckdb(&source_path);
24304 let mut source = block_on(service.open_duckdb_source(OpenDuckDbSourceRequest {
24305 path: source_path.clone(),
24306 }))
24307 .expect("DuckDB SQL source should open");
24308 let analysis = block_on(service.analyze_sql_source_metadata(
24309 &mut source,
24310 AnalyzeSqlSourceRequest {
24311 sql: "SELECT death_id, site_id, cause FROM deaths".to_string(),
24312 },
24313 ))
24314 .expect("SQL source metadata should analyze");
24315 assert_eq!(analysis.columns.len(), 3);
24316 assert_eq!(analysis.columns[0].name, "death_id");
24317 assert_eq!(
24318 analysis.columns[2]
24319 .vocabulary
24320 .as_ref()
24321 .map(|vocabulary| vocabulary.items.len()),
24322 Some(2)
24323 );
24324
24325 let mut session = test_lake_session(&lake_root);
24326 let materialized = block_on(service.sql_to_dataset(
24327 &mut session,
24328 &mut source,
24329 SqlToDatasetRequest {
24330 classification: ahri_tre_types::IngestClassification::derived_high(),
24331 study_id: study.study_id,
24332 domain_id: domain.domain_id,
24333 dataset_asset_id: None,
24334 dataset_name: nc_name("p1_14_sql_dataset"),
24335 dataset_version_id: None,
24336 sql: "SELECT death_id, site_id, cause FROM deaths".to_string(),
24337 description: Some("Dataset loaded from a SQL source".to_string()),
24338 agent_instructions: None,
24339 version_note: Some("Initial SQL dataset load".to_string()),
24340 transformation: test_transformation(
24341 99,
24342 TransformationType::Ingest,
24343 "ingest SQL source to dataset",
24344 ),
24345 force_metadata_updates: false,
24346 },
24347 ))
24348 .expect("sql-to-dataset should succeed");
24349
24350 assert_eq!(materialized.row_count, 3);
24351 assert_eq!(materialized.column_count, 3);
24352 assert!(materialized.lineage.inputs.is_empty());
24353 assert_eq!(materialized.variables.len(), 3);
24354 let cause = materialized
24355 .variables
24356 .iter()
24357 .find(|registered| registered.variable.name.as_str() == "cause")
24358 .expect("cause variable should be registered");
24359 assert_eq!(cause.variable.value_type_id, ValueTypeId(7));
24360 assert_eq!(cause.vocabulary_items.len(), 2);
24361 assert_eq!(
24362 dataset_row_count(
24363 &session,
24364 &materialized.lake_schema,
24365 &materialized.lake_table
24366 ),
24367 3
24368 );
24369
24370 let _ = fs::remove_dir_all(&lake_root);
24371 }
24372
24373 #[test]
24374 fn dataset_sql_transform_refuses_an_unconfigured_capture_connection() {
24375 let repositories = AssetWorkflowRepositories::new();
24376 let service = repositories.service();
24377 let study = test_study("p1_14_sql_transform_study");
24378 let domain = test_domain(90_114, "p1_14_sql_transform_domain");
24379 repositories.seed_study(study.clone());
24380 repositories.seed_study_domain(study.study_id, domain.clone());
24381 let lake_root = temp_lake_root_path();
24382 let source_path = lake_root.join("meta_transform_test.duckdb");
24383 seed_meta_test_duckdb(&source_path);
24384 let mut source = block_on(service.open_duckdb_source(OpenDuckDbSourceRequest {
24385 path: source_path.clone(),
24386 }))
24387 .expect("DuckDB SQL source should open");
24388 let mut session = test_lake_session(&lake_root);
24389 let source_dataset = block_on(service.sql_to_dataset(
24390 &mut session,
24391 &mut source,
24392 SqlToDatasetRequest {
24393 classification: ahri_tre_types::IngestClassification::derived_high(),
24394 study_id: study.study_id,
24395 domain_id: domain.domain_id,
24396 dataset_asset_id: None,
24397 dataset_name: nc_name("p1_14_sql_transform_source"),
24398 dataset_version_id: None,
24399 sql: "SELECT death_id, site_id, cause FROM deaths".to_string(),
24400 description: Some("Dataset loaded from a SQL source".to_string()),
24401 agent_instructions: None,
24402 version_note: Some("Initial SQL dataset load".to_string()),
24403 transformation: test_transformation(
24404 199,
24405 TransformationType::Ingest,
24406 "ingest SQL source to dataset",
24407 ),
24408 force_metadata_updates: false,
24409 },
24410 ))
24411 .expect("source dataset should materialize");
24412 let error = block_on(service.transform_dataset_with_lake_sql(
24413 &mut session,
24414 DatasetSqlTransformRequest {
24415 study_id: study.study_id,
24416 dataset_asset_id: None,
24417 dataset_name: nc_name("p1_14_sql_transform_summary"),
24418 dataset_version_id: None,
24419 inputs: std::collections::BTreeMap::from([("source".into(), source_dataset.dataset.dataset_id)]),
24420 views: Default::default(),
24421 budgets: Default::default(),
24422 sql: "SELECT CAST(site_id AS VARCHAR) AS redcap_event, COUNT(*) AS record_count FROM source GROUP BY 1 ORDER BY 1".into(),
24423 description: Some("Dataset SQL transform summary".to_string()),
24424 agent_instructions: None,
24425 version_note: Some("Summary transform".to_string()),
24426 transformation: test_transformation(
24427 200,
24428 TransformationType::Transform,
24429 "summarize source dataset",
24430 ),
24431 },
24432 ))
24433 .expect_err("an unconfigured connection cannot run SQL against its Lake authority");
24434 assert!(matches!(error, AppError::Infrastructure(_)));
24435 let _ = fs::remove_dir_all(&lake_root);
24436 }
24437
24438 #[test]
24439 fn sql_to_dataset_loads_sqlite_source_when_extension_available() {
24440 let repositories = AssetWorkflowRepositories::new();
24441 let service = repositories.service();
24442 let study = test_study("p1_14_sqlite_dataset_study");
24443 let domain = test_domain(90_015, "p1_14_sqlite_domain");
24444 repositories.seed_study(study.clone());
24445 repositories.seed_study_domain(study.study_id, domain.clone());
24446 let lake_root = temp_lake_root_path();
24447 let source_path = lake_root.join("meta_test.sqlite");
24448 if let Err(error) = seed_meta_test_sqlite(&source_path) {
24449 eprintln!(
24450 "skipping SQLite SQL-source ingest test; DuckDB sqlite extension unavailable: {error}"
24451 );
24452 let _ = fs::remove_dir_all(&lake_root);
24453 return;
24454 }
24455 let mut source =
24456 block_on(service.open_sqlite_source(OpenSqliteSourceRequest { path: source_path }))
24457 .expect("SQLite SQL source should open");
24458 let mut session = test_lake_session(&lake_root);
24459
24460 let materialized = block_on(service.sql_to_dataset(
24461 &mut session,
24462 &mut source,
24463 SqlToDatasetRequest {
24464 classification: ahri_tre_types::IngestClassification::derived_high(),
24465 study_id: study.study_id,
24466 domain_id: domain.domain_id,
24467 dataset_asset_id: None,
24468 dataset_name: nc_name("p1_14_sqlite_dataset"),
24469 dataset_version_id: None,
24470 sql: "SELECT death_id, site_id, cause FROM deaths".to_string(),
24471 description: Some("Dataset loaded from a SQLite source".to_string()),
24472 agent_instructions: None,
24473 version_note: Some("Initial SQLite SQL dataset load".to_string()),
24474 transformation: test_transformation(
24475 102,
24476 TransformationType::Ingest,
24477 "ingest SQLite SQL source to dataset",
24478 ),
24479 force_metadata_updates: false,
24480 },
24481 ))
24482 .expect("SQLite sql-to-dataset should succeed when sqlite extension is available");
24483
24484 assert_eq!(materialized.row_count, 3);
24485 assert!(materialized.lineage.inputs.is_empty());
24486 assert_eq!(materialized.variables.len(), 3);
24487 assert!(
24488 materialized
24489 .variables
24490 .iter()
24491 .any(|registered| registered.variable.name.as_str() == "cause"
24492 && registered.variable.value_type_id == ValueTypeId(7))
24493 );
24494
24495 let _ = fs::remove_dir_all(&lake_root);
24496 }
24497
24498 #[test]
24499 fn redcap_offline_ingest_preserves_artifacts_and_materializes_form_dataset() {
24500 let repositories = AssetWorkflowRepositories::new();
24501 let service = repositories.service();
24502 let study = test_study("p2_1_redcap_study");
24503 let domain = test_domain(90_001, "p2_1_redcap_domain");
24504 repositories.seed_study(study.clone());
24505 repositories.seed_study_domain(study.study_id, domain.clone());
24506 let project_info = workspace_file("data/redcap_api/project_info.json");
24507 let metadata_raw = workspace_file("data/redcap_api/metadata_raw.json");
24508 let records_eav = workspace_file("data/redcap_api/records_eav.csv");
24509 assert!(project_info.is_file());
24510 assert!(metadata_raw.is_file());
24511 assert!(records_eav.is_file());
24512 let lake_root = temp_lake_root_path();
24513 let mut session = test_lake_session(&lake_root);
24514
24515 let result = block_on(service.ingest_redcap_offline(
24516 &mut session,
24517 OfflineRedcapIngestRequest {
24518 no_register_dictionary: false,
24519 classification: ahri_tre_types::IngestClassification::derived_high(),
24520 study_id: Some(study.study_id),
24521 domain_id: Some(domain.domain_id),
24522 domain_name: None,
24523 study_name: None,
24524 project_info_path: project_info.display().to_string(),
24525 metadata_path: metadata_raw.display().to_string(),
24526 records_eav_path: records_eav.display().to_string(),
24527 forms: Vec::new(),
24528 skip_forms: Vec::new(),
24529 dataset_prefix: None,
24530 description: Some("Offline REDCap fixture ingest".to_string()),
24531 agent_instructions: None,
24532 version_note: Some("Initial REDCap offline ingest".to_string()),
24533 ingest_transformation: test_transformation(
24534 121,
24535 TransformationType::Ingest,
24536 "ingest REDCap offline artifacts",
24537 ),
24538 transform_transformation: test_transformation(
24539 122,
24540 TransformationType::Transform,
24541 "materialize REDCap form dataset",
24542 ),
24543 force_metadata_updates: false,
24544 compress: true,
24545 encrypt: None,
24546 },
24547 ))
24548 .expect("offline REDCap ingest should succeed");
24549
24550 assert_eq!(result.study.study_id, study.study_id);
24551 assert_eq!(result.datafiles.len(), 3);
24552 assert!(result.ingest_lineage.inputs.is_empty());
24553 assert_eq!(result.ingest_lineage.outputs.len(), 3);
24554 assert_eq!(result.datasets.len(), 1);
24555 let dataset = &result.datasets[0];
24556 assert_eq!(dataset.catalog.asset.asset_type, AssetType::Dataset);
24557 assert_eq!(
24558 dataset.catalog.asset.name.as_str(),
24559 "redcap_1194_cohort_survey"
24560 );
24561 assert!(dataset.row_count > 0);
24562 assert!(dataset.variables.iter().any(|registered| {
24563 registered.variable.name.as_str() == "cs_coh_countries"
24564 && registered.variable.value_type_id == ValueTypeId(8)
24565 && registered
24566 .vocabulary_items
24567 .iter()
24568 .any(|item| item.code == "ZAF")
24569 }));
24570 assert!(dataset.lineage.inputs.iter().all(|input| {
24571 result
24572 .datafiles
24573 .iter()
24574 .any(|datafile| datafile.datafile.datafile_id == input.version_id)
24575 }));
24576 let count = dataset_row_count(&session, &dataset.lake_schema, &dataset.lake_table);
24577 assert_eq!(count as usize, dataset.row_count);
24578
24579 let _ = fs::remove_dir_all(&lake_root);
24580 }
24581
24582 #[test]
24583 fn datafile_to_dataset_materializes_lake_datafile_with_transform_provenance() {
24584 let repositories = AssetWorkflowRepositories::new();
24585 let service = repositories.service();
24586 let study = test_study("p1_13_datafile_dataset_study");
24587 repositories.seed_study(study.clone());
24588 let domain = test_domain(90_301, "p1_13_datafile_dataset_domain");
24589 repositories.seed_study_domain(study.study_id, domain.clone());
24590 let source_path = workspace_file("data/ingest.csv");
24591 let lake_root = temp_lake_root_path();
24592 let ingest_transformation =
24593 test_transformation(97, TransformationType::Ingest, "ingest csv as datafile");
24594 let ingested = block_on(service.ingest_file(IngestFileRequest {
24595 classification: ahri_tre_types::IngestClassification::derived_high(),
24596 study_id: study.study_id,
24597 asset_id: None,
24598 asset_name: nc_name("p1_13_managed_datafile"),
24599 version_id: None,
24600 source: IngestFileSource::LocalPath {
24601 path: source_path.display().to_string(),
24602 },
24603 lake_root: lake_root.display().to_string(),
24604 edam_format: Some("EDAM:format_3752".to_string()),
24605 compress: true,
24606 encrypt: None,
24607 description: Some("Managed lake datafile".to_string()),
24608 agent_instructions: None,
24609 version_note: Some("Initial managed datafile".to_string()),
24610 transformation: ingest_transformation,
24611 }))
24612 .expect("source datafile ingest should succeed");
24613 let mut session = test_lake_session(&lake_root);
24614 let transform_transformation = test_transformation(
24615 98,
24616 TransformationType::Transform,
24617 "transform datafile to dataset",
24618 );
24619
24620 let request = DataFileToDatasetRequest {
24621 study_id: study.study_id,
24622 metadata_domain_id: None,
24623 dataset_asset_id: None,
24624 dataset_name: nc_name("p1_13_datafile_dataset"),
24625 dataset_version_id: None,
24626 source_datafile_id: ingested.datafile.datafile_id,
24627 input_format: Some("csv".to_string()),
24628 header: None,
24629 delimiter: None,
24630 null_strings: Vec::new(),
24631 json_format: None,
24632 sheet: None,
24633 description: Some("Dataset loaded from a managed datafile".to_string()),
24634 agent_instructions: None,
24635 version_note: Some("Initial datafile-derived dataset".to_string()),
24636 transformation: transform_transformation.clone(),
24637 strict_metadata: false,
24638 sample_rows: None,
24639 vocabulary_threshold: None,
24640 force_metadata_updates: false,
24641 variable_registrations: Vec::new(),
24642 };
24643 use super::datafile_materialization::{
24644 DatafileLifecycle, DatafileStage, ResolvedDatafileMaterialization,
24645 };
24646 struct StopAtStage {
24647 stop: DatafileStage,
24648 source: VersionId,
24649 domain: ahri_tre_types::DomainId,
24650 cleanup: bool,
24651 }
24652 impl DatafileLifecycle for StopAtStage {
24653 fn checkpoint(
24654 &mut self,
24655 stage: DatafileStage,
24656 resolved: &ResolvedDatafileMaterialization,
24657 ) -> Result<(), AppError> {
24658 assert_eq!(resolved.source_version.version_id, self.source);
24659 assert_eq!(resolved.request.metadata_domain_id, Some(self.domain));
24660 if stage == self.stop {
24661 Err(AppError::Infrastructure("fixture lifecycle stop".into()))
24662 } else {
24663 Ok(())
24664 }
24665 }
24666 fn cleanup_started(&mut self) {
24667 self.cleanup = true;
24668 }
24669 }
24670 for stop in [
24671 DatafileStage::BeforePreparation,
24672 DatafileStage::BeforeLoad,
24673 DatafileStage::AfterPreparation,
24674 DatafileStage::BeforeMetadataCommit,
24675 ] {
24676 let mut lifecycle = StopAtStage {
24677 stop,
24678 source: ingested.datafile.datafile_id,
24679 domain: domain.domain_id,
24680 cleanup: false,
24681 };
24682 let error = block_on(service.datafile_to_dataset_with_lifecycle(
24683 &mut session,
24684 request.clone(),
24685 &mut lifecycle,
24686 ))
24687 .unwrap_err();
24688 assert!(error.to_string().contains("fixture lifecycle stop"));
24689 assert!(lifecycle.cleanup);
24690 assert!(
24691 block_on(
24692 service
24693 .assets
24694 .get_asset_by_name(study.study_id, request.dataset_name.as_str())
24695 )
24696 .unwrap()
24697 .is_none()
24698 );
24699 let relation = ahri_tre_lake::dataset_loading::dataset_table_relation(
24700 study.study_id,
24701 request.dataset_name.as_str(),
24702 1,
24703 0,
24704 0,
24705 );
24706 assert!(!dataset_table_exists(
24707 &session,
24708 &relation.schema_name,
24709 &relation.table_name
24710 ));
24711 assert_eq!(
24712 fs::read_dir(lake_root.with_extension("scratch"))
24713 .unwrap()
24714 .count(),
24715 1
24716 );
24717 }
24718 let mut no_scratch = DataStoreSession::new_lake_only_for_test(
24719 test_runtime(&lake_root),
24720 duckdb::Connection::open_in_memory().unwrap(),
24721 );
24722 let error = block_on(service.datafile_to_dataset(&mut no_scratch, request.clone()))
24723 .expect_err("Dataset materialization requires configured scratch");
24724 assert!(error.to_string().contains("scratch"), "{error}");
24725 assert!(
24726 block_on(
24727 service
24728 .assets
24729 .get_asset_by_name(study.study_id, "p1_13_datafile_dataset")
24730 )
24731 .unwrap()
24732 .is_none()
24733 );
24734
24735 let materialized = block_on(service.datafile_to_dataset(&mut session, request))
24736 .expect("datafile-to-dataset should succeed");
24737
24738 assert_eq!(materialized.catalog.asset.asset_type, AssetType::Dataset);
24739 assert_eq!(materialized.row_count, 10);
24740 assert_eq!(materialized.column_count, 5);
24741 assert_eq!(materialized.variables.len(), 5);
24742 assert_eq!(materialized.metadata_warnings.len(), 5);
24743 assert!(
24744 materialized
24745 .metadata_warnings
24746 .iter()
24747 .any(|warning| warning.contains("automatically registered CSV column year"))
24748 );
24749 let registered = materialized
24750 .variables
24751 .iter()
24752 .map(|registered| {
24753 (
24754 registered.variable.name.as_str(),
24755 registered.variable.value_type_id,
24756 registered.vocabulary.is_some(),
24757 registered.vocabulary_items.len(),
24758 registered.link.dataset_id,
24759 )
24760 })
24761 .collect::<Vec<_>>();
24762 assert_eq!(
24763 registered,
24764 vec![
24765 (
24766 "category",
24767 ValueTypeId(7),
24768 true,
24769 4,
24770 materialized.dataset.dataset_id
24771 ),
24772 (
24773 "country",
24774 ValueTypeId(7),
24775 true,
24776 10,
24777 materialized.dataset.dataset_id
24778 ),
24779 (
24780 "year",
24781 ValueTypeId(1),
24782 false,
24783 0,
24784 materialized.dataset.dataset_id
24785 ),
24786 (
24787 "population",
24788 ValueTypeId(1),
24789 false,
24790 0,
24791 materialized.dataset.dataset_id
24792 ),
24793 (
24794 "date_added",
24795 ValueTypeId(7),
24796 true,
24797 8,
24798 materialized.dataset.dataset_id
24799 ),
24800 ]
24801 );
24802 assert_eq!(materialized.lake_table, "p1_13_datafile_dataset_1_0_0");
24803 assert_eq!(
24804 materialized.lineage.transformation.transformation_type,
24805 TransformationType::Transform
24806 );
24807 assert_eq!(
24808 materialized
24809 .lineage
24810 .inputs
24811 .iter()
24812 .map(|input| input.version_id)
24813 .collect::<Vec<_>>(),
24814 vec![ingested.datafile.datafile_id]
24815 );
24816 assert_eq!(
24817 materialized
24818 .lineage
24819 .outputs
24820 .iter()
24821 .map(|output| output.version_id)
24822 .collect::<Vec<_>>(),
24823 vec![materialized.dataset.dataset_id]
24824 );
24825 assert_eq!(
24826 dataset_row_count(
24827 &session,
24828 &materialized.lake_schema,
24829 &materialized.lake_table
24830 ),
24831 10
24832 );
24833 assert!(
24834 !lake_root
24835 .join("__tre_duckdb_stage")
24836 .join(format!(
24837 "materialized-{}.csv",
24838 ingested.datafile.datafile_id.0
24839 ))
24840 .exists()
24841 );
24842
24843 let _ = fs::remove_dir_all(&lake_root);
24844 }
24845
24846 #[test]
24847 fn datafile_to_dataset_preserves_explicit_variable_registrations_over_csv_auto() {
24848 let repositories = AssetWorkflowRepositories::new();
24849 let service = repositories.service();
24850 let study = test_study("p1_13_explicit_csv_metadata_study");
24851 repositories.seed_study(study.clone());
24852 let domain = test_domain(90_302, "p1_13_explicit_csv_metadata_domain");
24853 repositories.seed_study_domain(study.study_id, domain.clone());
24854 let source_path = workspace_file("data/ingest.csv");
24855 let lake_root = temp_lake_root_path();
24856 let ingested = block_on(service.ingest_file(IngestFileRequest {
24857 classification: ahri_tre_types::IngestClassification::derived_high(),
24858 study_id: study.study_id,
24859 asset_id: None,
24860 asset_name: nc_name("p1_13_explicit_csv_metadata_file"),
24861 version_id: None,
24862 source: IngestFileSource::LocalPath {
24863 path: source_path.display().to_string(),
24864 },
24865 lake_root: lake_root.display().to_string(),
24866 edam_format: Some("EDAM:format_3752".to_string()),
24867 compress: true,
24868 encrypt: None,
24869 description: Some("Managed CSV datafile".to_string()),
24870 agent_instructions: None,
24871 version_note: Some("Initial managed datafile".to_string()),
24872 transformation: test_transformation(
24873 99,
24874 TransformationType::Ingest,
24875 "ingest csv for explicit metadata precedence",
24876 ),
24877 }))
24878 .expect("source datafile ingest should succeed");
24879 let mut session = test_lake_session(&lake_root);
24880
24881 let materialized = block_on(service.datafile_to_dataset(
24882 &mut session,
24883 DataFileToDatasetRequest {
24884 study_id: study.study_id,
24885 metadata_domain_id: None,
24886 dataset_asset_id: None,
24887 dataset_name: nc_name("p1_13_explicit_csv_metadata_dataset"),
24888 dataset_version_id: None,
24889 source_datafile_id: ingested.datafile.datafile_id,
24890 input_format: Some("csv".to_string()),
24891 header: None,
24892 delimiter: None,
24893 null_strings: Vec::new(),
24894 json_format: None,
24895 sheet: None,
24896 description: Some("Dataset loaded with explicit metadata".to_string()),
24897 agent_instructions: None,
24898 version_note: Some("Initial datafile-derived dataset".to_string()),
24899 transformation: test_transformation(
24900 100,
24901 TransformationType::Transform,
24902 "transform datafile with explicit metadata",
24903 ),
24904 strict_metadata: false,
24905 sample_rows: None,
24906 vocabulary_threshold: None,
24907 force_metadata_updates: false,
24908 variable_registrations: vec![DatasetVariableRegistrationRequest {
24909 variable: test_variable(
24910 10_302,
24911 domain.domain_id.0,
24912 "country",
24913 KeyRole::Record,
24914 None,
24915 ),
24916 row_role: Some(KeyRole::External),
24917 vocabulary: None,
24918 vocabulary_items: Vec::new(),
24919 }],
24920 },
24921 ))
24922 .expect("datafile-to-dataset should preserve explicit registrations");
24923
24924 assert_eq!(materialized.column_count, 5);
24925 assert_eq!(materialized.variables.len(), 1);
24926 assert_eq!(materialized.variables[0].variable.name.as_str(), "country");
24927 assert_eq!(materialized.variables[0].variable.key_role, KeyRole::Record);
24928 assert_eq!(materialized.variables[0].link.row_role, KeyRole::External);
24929 assert_eq!(materialized.metadata_warnings.len(), 1);
24930 assert!(
24931 materialized.metadata_warnings[0]
24932 .contains("explicit variable registrations supplied; ignored automatic/discovered CSV metadata for 5 imported columns"),
24933 "{:?}",
24934 materialized.metadata_warnings
24935 );
24936 assert!(
24937 !materialized
24938 .metadata_warnings
24939 .iter()
24940 .any(|warning| warning.contains("automatically registered CSV column")),
24941 "{:?}",
24942 materialized.metadata_warnings
24943 );
24944
24945 let _ = fs::remove_dir_all(&lake_root);
24946 }
24947
24948 #[test]
24949 fn managed_datafile_derivation_accepts_pinned_reference_and_checks_scope() {
24950 let repositories = AssetWorkflowRepositories::new();
24951 let mut service = repositories.service();
24952 service.datastore_id = Some(uuid(11));
24953 let study = test_study("ticket11_pinned_metadata_study");
24954 repositories.seed_study(study.clone());
24955 let ignored_domain = test_domain(90_312, "p1_13_ignored_csv_metadata_domain");
24956 let selected_domain = test_domain(90_313, "ticket11_pinned_metadata_domain");
24957 repositories.seed_study_domain(study.study_id, ignored_domain);
24958 repositories.seed_study_domain(study.study_id, selected_domain.clone());
24959 let source_path = workspace_file("data/ingest.json");
24960 let lake_root = temp_lake_root_path();
24961 let source = block_on(service.ingest_file(IngestFileRequest {
24962 classification: ahri_tre_types::IngestClassification::derived_high(),
24963 study_id: study.study_id,
24964 asset_id: None,
24965 asset_name: nc_name("ticket11_pinned_metadata_file"),
24966 version_id: None,
24967 source: IngestFileSource::LocalPath {
24968 path: source_path.display().to_string(),
24969 },
24970 lake_root: lake_root.display().to_string(),
24971 edam_format: Some("EDAM:format_3464".to_string()),
24972 compress: true,
24973 encrypt: None,
24974 description: Some("Managed CSV datafile".to_string()),
24975 agent_instructions: None,
24976 version_note: Some("Initial managed datafile".to_string()),
24977 transformation: test_transformation(
24978 111,
24979 TransformationType::Ingest,
24980 "ingest csv for selected metadata domain",
24981 ),
24982 }))
24983 .expect("source datafile ingest should succeed");
24984 let mut session = test_lake_session(&lake_root);
24985
24986 let request = DeriveDatasetFromManagedDataFileRequest {
24987 study: StudySelector::Id {
24988 study_id: study.study_id,
24989 },
24990 metadata_domain: DomainSelector::Id {
24991 domain_id: selected_domain.domain_id,
24992 },
24993 dataset_name: nc_name("ticket11_pinned_metadata_dataset"),
24994 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
24995 asset: ahri_tre_protocol::asset::AssetSelector::Id {
24996 asset: ahri_tre_protocol::ingest::version_ref(
24997 ahri_tre_protocol::PublicUuid::from_uuid(uuid(11)),
24998 source.datafile.datafile_id,
24999 ),
25000 study: Some(ahri_tre_protocol::study::StudySelector::Name {
25001 domain: None,
25002 name: ahri_tre_protocol::PublicName::new(study.name.as_str()).unwrap(),
25003 }),
25004 asset_type: Some(AssetType::File),
25005 },
25006 version: None,
25007 },
25008 input_format: Some("json".to_string()),
25009 header: None,
25010 delimiter: None,
25011 null_strings: Vec::new(),
25012 json_format: None,
25013 sheet: None,
25014 description: Some("Dataset loaded with selected metadata domain".to_string()),
25015 version_note: Some("Initial datafile-derived dataset".to_string()),
25016 replace: false,
25017 new_version: None,
25018 transformation: test_transformation(
25019 112,
25020 TransformationType::Transform,
25021 "transform datafile with selected metadata domain",
25022 ),
25023 sample_rows: Some(50),
25024 vocabulary_threshold: Some(250),
25025 force_metadata_updates: false,
25026 };
25027 let other_study = test_study("ticket11_wrong_source_scope");
25028 repositories.seed_study(other_study.clone());
25029 for conflict in [
25030 "study",
25031 "datastore",
25032 "type",
25033 "version",
25034 "replace",
25035 "output_version",
25036 ] {
25037 let mut invalid = request.clone();
25038 if let ahri_tre_protocol::asset::AssetSelector::Id {
25039 asset,
25040 study,
25041 asset_type,
25042 } = &mut invalid.source.asset
25043 {
25044 match conflict {
25045 "study" => {
25046 *study = Some(ahri_tre_protocol::study::StudySelector::Name {
25047 domain: None,
25048 name: ahri_tre_protocol::PublicName::new(other_study.name.as_str())
25049 .unwrap(),
25050 })
25051 }
25052 "datastore" => {
25053 asset.datastore_id = ahri_tre_protocol::PublicUuid::from_uuid(uuid(12))
25054 }
25055 "type" => *asset_type = Some(AssetType::Dataset),
25056 "version" => invalid.source.version = Some("99.0.0".into()),
25057 "replace" => invalid.replace = true,
25058 "output_version" => invalid.new_version = Some("2.0.0".into()),
25059 _ => unreachable!(),
25060 }
25061 }
25062 assert!(
25063 block_on(service.derive_dataset_from_managed_datafile(&mut session, invalid))
25064 .is_err(),
25065 "{conflict}"
25066 );
25067 }
25068 let derived = block_on(service.derive_dataset_from_managed_datafile(&mut session, request))
25069 .expect("pinned Datafile source should materialize");
25070 assert_eq!(derived.study.study_id, study.study_id);
25071 let materialized = derived.materialization;
25072
25073 assert_eq!(materialized.dataset.row_count, 10);
25074 assert_eq!(materialized.dataset.column_count, 5);
25075 assert_eq!(
25076 materialized.dataset.lineage.inputs[0].version_id,
25077 source.datafile.datafile_id
25078 );
25079 assert_eq!(materialized.dataset.variables.len(), 5);
25080 assert!(
25081 materialized
25082 .dataset
25083 .variables
25084 .iter()
25085 .all(|variable| variable.variable.domain_id == selected_domain.domain_id)
25086 );
25087
25088 let metadata = block_on(service.read_dataset_metadata(ReadDatasetMetadataRequest {
25089 dataset: DatasetMetadataSelector::Name {
25090 study: StudySelector::Id {
25091 study_id: study.study_id,
25092 },
25093 name: nc_name("ticket11_pinned_metadata_dataset"),
25094 },
25095 with_variables: true,
25096 }))
25097 .expect("dataset metadata should resolve")
25098 .expect("dataset metadata should exist");
25099 assert_eq!(metadata.study.study_id, study.study_id);
25100 assert_eq!(metadata.metadata.variables.len(), 5);
25101
25102 let _ = fs::remove_dir_all(&lake_root);
25103 }
25104
25105 #[test]
25106 fn managed_datafile_derivation_accepts_tabular_formats_and_preserves_provenance() {
25107 for (format, declared) in [
25108 ("json", "EDAM:format_3464"),
25109 ("arrow", "application/vnd.apache.arrow.file"),
25110 ("parquet", "application/vnd.apache.parquet"),
25111 (
25112 "xlsx",
25113 "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
25114 ),
25115 ] {
25116 let repositories = AssetWorkflowRepositories::new();
25117 let service = repositories.service();
25118 let study = test_study("ticket11_json_metadata_study");
25119 repositories.seed_study(study.clone());
25120 let ignored_domain = test_domain(90_312, "p1_13_ignored_csv_metadata_domain");
25121 let selected_domain = test_domain(90_313, "ticket11_json_metadata_domain");
25122 repositories.seed_study_domain(study.study_id, ignored_domain);
25123 repositories.seed_study_domain(study.study_id, selected_domain.clone());
25124 let lake_root = temp_lake_root_path();
25125 let source_path = if format == "xlsx" {
25126 let path = lake_root.join("source.xlsx");
25127 write_minimal_xlsx_fixture(&path).expect("valid XLSX fixture");
25128 path
25129 } else {
25130 workspace_file(&format!("data/ingest.{format}"))
25131 };
25132 let (rows, columns) = if format == "xlsx" { (2, 3) } else { (10, 5) };
25133 block_on(service.ingest_file(IngestFileRequest {
25134 classification: ahri_tre_types::IngestClassification::derived_high(),
25135 study_id: study.study_id,
25136 asset_id: None,
25137 asset_name: nc_name("ticket11_json_metadata_file"),
25138 version_id: None,
25139 source: IngestFileSource::LocalPath {
25140 path: source_path.display().to_string(),
25141 },
25142 lake_root: lake_root.display().to_string(),
25143 edam_format: Some(declared.to_string()),
25144 compress: true,
25145 encrypt: None,
25146 description: Some("Managed CSV datafile".to_string()),
25147 agent_instructions: None,
25148 version_note: Some("Initial managed datafile".to_string()),
25149 transformation: test_transformation(
25150 111,
25151 TransformationType::Ingest,
25152 "ingest csv for selected metadata domain",
25153 ),
25154 }))
25155 .expect("source datafile ingest should succeed");
25156 let mut session = test_lake_session(&lake_root);
25157
25158 let derived = block_on(
25159 service.derive_dataset_from_managed_datafile(
25160 &mut session,
25161 DeriveDatasetFromManagedDataFileRequest {
25162 study: StudySelector::Id {
25163 study_id: study.study_id,
25164 },
25165 metadata_domain: DomainSelector::Id {
25166 domain_id: selected_domain.domain_id,
25167 },
25168 dataset_name: nc_name("ticket11_json_metadata_dataset"),
25169 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
25170 asset: ahri_tre_protocol::asset::AssetSelector::Name {
25171 study: ahri_tre_protocol::study::StudySelector::Name {
25172 domain: None,
25173 name: ahri_tre_protocol::PublicName::new(study.name.as_str())
25174 .unwrap(),
25175 },
25176 name: ahri_tre_protocol::PublicName::new(
25177 "ticket11_json_metadata_file",
25178 )
25179 .unwrap(),
25180 asset_type: Some(AssetType::File),
25181 },
25182 version: Some("latest".to_string()),
25183 },
25184 input_format: Some(format.to_string()),
25185 header: None,
25186 delimiter: None,
25187 null_strings: Vec::new(),
25188 json_format: None,
25189 sheet: None,
25190 description: Some(
25191 "Dataset loaded with selected metadata domain".to_string(),
25192 ),
25193 version_note: Some("Initial datafile-derived dataset".to_string()),
25194 replace: false,
25195 new_version: None,
25196 transformation: test_transformation(
25197 112,
25198 TransformationType::Transform,
25199 "transform datafile with selected metadata domain",
25200 ),
25201 sample_rows: Some(50),
25202 vocabulary_threshold: Some(250),
25203 force_metadata_updates: false,
25204 },
25205 ),
25206 )
25207 .expect("managed datafile materialization should use selected metadata domain");
25208 assert_eq!(derived.study.study_id, study.study_id);
25209 let materialized = derived.materialization;
25210
25211 assert_eq!(materialized.dataset.row_count, rows);
25212 assert_eq!(materialized.dataset.column_count, columns);
25213 assert_eq!(materialized.dataset.lineage.inputs.len(), 1);
25214 assert_eq!(materialized.dataset.variables.len(), columns);
25215 assert!(
25216 materialized
25217 .dataset
25218 .variables
25219 .iter()
25220 .all(|variable| variable.variable.domain_id == selected_domain.domain_id)
25221 );
25222
25223 let metadata = block_on(service.read_dataset_metadata(ReadDatasetMetadataRequest {
25224 dataset: DatasetMetadataSelector::Name {
25225 study: StudySelector::Id {
25226 study_id: study.study_id,
25227 },
25228 name: nc_name("ticket11_json_metadata_dataset"),
25229 },
25230 with_variables: true,
25231 }))
25232 .expect("dataset metadata should resolve")
25233 .expect("dataset metadata should exist");
25234 assert_eq!(metadata.study.study_id, study.study_id);
25235 assert_eq!(metadata.metadata.variables.len(), columns);
25236
25237 let _ = fs::remove_dir_all(&lake_root);
25238 }
25239 }
25240
25241 #[test]
25242 fn managed_datafile_derivation_workflow_resolves_scope_and_uses_selected_metadata_domain() {
25243 let repositories = AssetWorkflowRepositories::new();
25244 let service = repositories.service();
25245 let study = test_study("p1_13_selected_csv_metadata_study");
25246 repositories.seed_study(study.clone());
25247 let ignored_domain = test_domain(90_312, "p1_13_ignored_csv_metadata_domain");
25248 let selected_domain = test_domain(90_313, "p1_13_selected_csv_metadata_domain");
25249 repositories.seed_study_domain(study.study_id, ignored_domain);
25250 repositories.seed_study_domain(study.study_id, selected_domain.clone());
25251 let source_path = workspace_file("data/ingest.csv");
25252 let lake_root = temp_lake_root_path();
25253 block_on(service.ingest_file(IngestFileRequest {
25254 classification: ahri_tre_types::IngestClassification::derived_high(),
25255 study_id: study.study_id,
25256 asset_id: None,
25257 asset_name: nc_name("p1_13_selected_csv_metadata_file"),
25258 version_id: None,
25259 source: IngestFileSource::LocalPath {
25260 path: source_path.display().to_string(),
25261 },
25262 lake_root: lake_root.display().to_string(),
25263 edam_format: Some("EDAM:format_3752".to_string()),
25264 compress: true,
25265 encrypt: None,
25266 description: Some("Managed CSV datafile".to_string()),
25267 agent_instructions: None,
25268 version_note: Some("Initial managed datafile".to_string()),
25269 transformation: test_transformation(
25270 111,
25271 TransformationType::Ingest,
25272 "ingest csv for selected metadata domain",
25273 ),
25274 }))
25275 .expect("source datafile ingest should succeed");
25276 let mut session = test_lake_session(&lake_root);
25277
25278 let derived = block_on(
25279 service.derive_dataset_from_managed_datafile(
25280 &mut session,
25281 DeriveDatasetFromManagedDataFileRequest {
25282 study: StudySelector::Id {
25283 study_id: study.study_id,
25284 },
25285 metadata_domain: DomainSelector::Id {
25286 domain_id: selected_domain.domain_id,
25287 },
25288 dataset_name: nc_name("p1_13_selected_csv_metadata_dataset"),
25289 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
25290 asset: ahri_tre_protocol::asset::AssetSelector::Name {
25291 study: ahri_tre_protocol::study::StudySelector::Name {
25292 domain: None,
25293 name: ahri_tre_protocol::PublicName::new(study.name.as_str())
25294 .unwrap(),
25295 },
25296 name: ahri_tre_protocol::PublicName::new(
25297 "p1_13_selected_csv_metadata_file",
25298 )
25299 .unwrap(),
25300 asset_type: Some(AssetType::File),
25301 },
25302 version: Some("latest".to_string()),
25303 },
25304 input_format: Some("csv".to_string()),
25305 header: None,
25306 delimiter: None,
25307 null_strings: Vec::new(),
25308 json_format: None,
25309 sheet: None,
25310 description: Some("Dataset loaded with selected metadata domain".to_string()),
25311 version_note: Some("Initial datafile-derived dataset".to_string()),
25312 replace: false,
25313 new_version: None,
25314 transformation: test_transformation(
25315 112,
25316 TransformationType::Transform,
25317 "transform datafile with selected metadata domain",
25318 ),
25319 sample_rows: Some(50),
25320 vocabulary_threshold: Some(250),
25321 force_metadata_updates: false,
25322 },
25323 ),
25324 )
25325 .expect("managed datafile materialization should use selected metadata domain");
25326 assert_eq!(derived.study.study_id, study.study_id);
25327 let materialized = derived.materialization;
25328
25329 assert_eq!(materialized.dataset.column_count, 5);
25330 assert_eq!(materialized.dataset.variables.len(), 5);
25331 assert!(
25332 materialized
25333 .dataset
25334 .variables
25335 .iter()
25336 .all(|variable| variable.variable.domain_id == selected_domain.domain_id)
25337 );
25338
25339 let metadata = block_on(service.read_dataset_metadata(ReadDatasetMetadataRequest {
25340 dataset: DatasetMetadataSelector::Name {
25341 study: StudySelector::Id {
25342 study_id: study.study_id,
25343 },
25344 name: nc_name("p1_13_selected_csv_metadata_dataset"),
25345 },
25346 with_variables: true,
25347 }))
25348 .expect("dataset metadata should resolve")
25349 .expect("dataset metadata should exist");
25350 assert_eq!(metadata.study.study_id, study.study_id);
25351 assert_eq!(metadata.metadata.variables.len(), 5);
25352
25353 block_on(service.ingest_file(IngestFileRequest {
25354 classification: ahri_tre_types::IngestClassification::derived_high(),
25355 study_id: study.study_id,
25356 asset_id: None,
25357 asset_name: nc_name("p1_13_non_csv_managed_file"),
25358 version_id: None,
25359 source: IngestFileSource::LocalPath {
25360 path: source_path.display().to_string(),
25361 },
25362 lake_root: lake_root.display().to_string(),
25363 edam_format: Some("EDAM:format_3464".to_string()),
25364 compress: true,
25365 encrypt: None,
25366 description: Some("Managed non-CSV datafile".to_string()),
25367 agent_instructions: None,
25368 version_note: Some("Non-CSV managed source".to_string()),
25369 transformation: test_transformation(
25370 116,
25371 TransformationType::Ingest,
25372 "ingest non-csv source for rejection",
25373 ),
25374 }))
25375 .expect("non-CSV source fixture should ingest as a managed Datafile");
25376 let error = block_on(
25377 service.derive_dataset_from_managed_datafile(
25378 &mut session,
25379 DeriveDatasetFromManagedDataFileRequest {
25380 study: StudySelector::Id {
25381 study_id: study.study_id,
25382 },
25383 metadata_domain: DomainSelector::Id {
25384 domain_id: selected_domain.domain_id,
25385 },
25386 dataset_name: nc_name("p1_13_rejected_non_csv_dataset"),
25387 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
25388 asset: ahri_tre_protocol::asset::AssetSelector::Name {
25389 study: ahri_tre_protocol::study::StudySelector::Name {
25390 domain: None,
25391 name: ahri_tre_protocol::PublicName::new(study.name.as_str())
25392 .unwrap(),
25393 },
25394 name: ahri_tre_protocol::PublicName::new("p1_13_non_csv_managed_file")
25395 .unwrap(),
25396 asset_type: Some(AssetType::File),
25397 },
25398 version: Some("latest".to_string()),
25399 },
25400 input_format: Some("csv".into()),
25401 header: None,
25402 delimiter: None,
25403 null_strings: Vec::new(),
25404 json_format: None,
25405 sheet: None,
25406 description: Some("Rejected non-CSV Dataset".to_string()),
25407 version_note: Some("Must not materialize".to_string()),
25408 replace: false,
25409 new_version: None,
25410 transformation: test_transformation(
25411 117,
25412 TransformationType::Transform,
25413 "reject non-csv managed source",
25414 ),
25415 sample_rows: None,
25416 vocabulary_threshold: None,
25417 force_metadata_updates: false,
25418 },
25419 ),
25420 )
25421 .expect_err("conflicting parser must fail before materialization");
25422 assert!(matches!(
25423 error,
25424 AppError::Validation(message)
25425 if message == "Datafile format and requested parser disagree"
25426 ));
25427 assert!(
25428 block_on(service.get_study_dataset(GetStudyDatasetRequest {
25429 study_id: study.study_id,
25430 dataset_name: nc_name("p1_13_rejected_non_csv_dataset"),
25431 with_variables: false,
25432 }))
25433 .expect("rejected Dataset readback should succeed")
25434 .is_none()
25435 );
25436
25437 let _ = fs::remove_dir_all(&lake_root);
25438 }
25439
25440 #[test]
25441 fn managed_datafile_to_dataset_rematerializes_after_only_dataset_version_withdrawal() {
25442 let repositories = AssetWorkflowRepositories::new();
25443 let service = repositories.service();
25444 let study = test_study("p1_13_withdraw_rematerialize_study");
25445 let domain = test_domain(90_314, "p1_13_withdraw_rematerialize_domain");
25446 repositories.seed_study(study.clone());
25447 repositories.seed_study_domain(study.study_id, domain.clone());
25448 repositories.grant_study_custodianship(study.study_id, "curator@example.test");
25449 let source_path = workspace_file("data/ingest.csv");
25450 let lake_root = temp_lake_root_path();
25451 block_on(service.ingest_file(IngestFileRequest {
25452 classification: ahri_tre_types::IngestClassification::derived_high(),
25453 study_id: study.study_id,
25454 asset_id: None,
25455 asset_name: nc_name("p1_13_withdraw_rematerialize_file"),
25456 version_id: None,
25457 source: IngestFileSource::LocalPath {
25458 path: source_path.display().to_string(),
25459 },
25460 lake_root: lake_root.display().to_string(),
25461 edam_format: Some("EDAM:format_3752".to_string()),
25462 compress: true,
25463 encrypt: None,
25464 description: Some("Managed CSV datafile".to_string()),
25465 agent_instructions: None,
25466 version_note: Some("Initial managed datafile".to_string()),
25467 transformation: test_transformation(
25468 113,
25469 TransformationType::Ingest,
25470 "ingest csv before dataset withdrawal rematerialization",
25471 ),
25472 }))
25473 .expect("source datafile ingest should succeed");
25474 let mut session = test_lake_session(&lake_root);
25475
25476 let first = block_on(
25477 service.derive_dataset_from_managed_datafile(
25478 &mut session,
25479 DeriveDatasetFromManagedDataFileRequest {
25480 study: StudySelector::Id {
25481 study_id: study.study_id,
25482 },
25483 metadata_domain: DomainSelector::Id {
25484 domain_id: domain.domain_id,
25485 },
25486 dataset_name: nc_name("p1_13_withdraw_rematerialize_dataset"),
25487 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
25488 asset: ahri_tre_protocol::asset::AssetSelector::Name {
25489 study: ahri_tre_protocol::study::StudySelector::Name {
25490 domain: None,
25491 name: ahri_tre_protocol::PublicName::new(study.name.as_str())
25492 .unwrap(),
25493 },
25494 name: ahri_tre_protocol::PublicName::new(
25495 "p1_13_withdraw_rematerialize_file",
25496 )
25497 .unwrap(),
25498 asset_type: Some(AssetType::File),
25499 },
25500 version: Some("latest".to_string()),
25501 },
25502 input_format: Some("csv".to_string()),
25503 header: None,
25504 delimiter: None,
25505 null_strings: Vec::new(),
25506 json_format: None,
25507 sheet: None,
25508 description: Some("Dataset loaded before withdrawal".to_string()),
25509 version_note: Some("Initial datafile-derived dataset".to_string()),
25510 replace: false,
25511 new_version: None,
25512 transformation: test_transformation(
25513 114,
25514 TransformationType::Transform,
25515 "transform datafile before dataset withdrawal",
25516 ),
25517 sample_rows: Some(50),
25518 vocabulary_threshold: Some(250),
25519 force_metadata_updates: false,
25520 },
25521 ),
25522 )
25523 .expect("initial managed datafile materialization should succeed")
25524 .materialization;
25525
25526 block_on(service.withdraw_dataset_version(
25527 None,
25528 WithdrawDatasetVersionRequest {
25529 study_id: study.study_id,
25530 dataset_name: first.dataset.catalog.asset.name.clone(),
25531 version: "1.0.0".to_string(),
25532 reason: "invalid source metadata".to_string(),
25533 actor: Some("curator@example.test".to_string()),
25534 force: true,
25535 drop_lake_table: false,
25536 },
25537 ))
25538 .expect("withdrawing the only dataset version should succeed");
25539 let listed_after_withdraw =
25540 block_on(service.list_study_datasets(ListStudyDatasetsRequest {
25541 study_id: study.study_id,
25542 include_versions: false,
25543 }))
25544 .expect("dataset list should remain readable after withdrawal");
25545 assert!(listed_after_withdraw.datasets.is_empty());
25546
25547 let replacement = block_on(
25548 service.derive_dataset_from_managed_datafile(
25549 &mut session,
25550 DeriveDatasetFromManagedDataFileRequest {
25551 study: StudySelector::Id {
25552 study_id: study.study_id,
25553 },
25554 metadata_domain: DomainSelector::Id {
25555 domain_id: domain.domain_id,
25556 },
25557 dataset_name: first.dataset.catalog.asset.name.clone(),
25558 source: ahri_tre_protocol::ingest::ManagedDataFileSource {
25559 asset: ahri_tre_protocol::asset::AssetSelector::Name {
25560 study: ahri_tre_protocol::study::StudySelector::Name {
25561 domain: None,
25562 name: ahri_tre_protocol::PublicName::new(study.name.as_str())
25563 .unwrap(),
25564 },
25565 name: ahri_tre_protocol::PublicName::new(
25566 "p1_13_withdraw_rematerialize_file",
25567 )
25568 .unwrap(),
25569 asset_type: Some(AssetType::File),
25570 },
25571 version: Some("latest".to_string()),
25572 },
25573 input_format: Some("csv".to_string()),
25574 header: None,
25575 delimiter: None,
25576 null_strings: Vec::new(),
25577 json_format: None,
25578 sheet: None,
25579 description: Some("Dataset reloaded after withdrawal".to_string()),
25580 version_note: Some("Replacement datafile-derived dataset".to_string()),
25581 replace: false,
25582 new_version: None,
25583 transformation: test_transformation(
25584 115,
25585 TransformationType::Transform,
25586 "transform datafile after dataset withdrawal",
25587 ),
25588 sample_rows: Some(50),
25589 vocabulary_threshold: Some(250),
25590 force_metadata_updates: false,
25591 },
25592 ),
25593 )
25594 .expect("replacement materialization should choose the next dataset version")
25595 .materialization;
25596
25597 assert_eq!(
25598 replacement.dataset.catalog.asset.asset_id,
25599 first.dataset.catalog.asset.asset_id
25600 );
25601 assert_eq!(
25602 replacement
25603 .dataset
25604 .catalog
25605 .latest_version
25606 .as_ref()
25607 .map(|version| { (version.major, version.minor, version.patch) }),
25608 Some((1, 1, 0))
25609 );
25610 let metadata_without_details =
25611 block_on(service.get_study_dataset(GetStudyDatasetRequest {
25612 study_id: study.study_id,
25613 dataset_name: first.dataset.catalog.asset.name.clone(),
25614 with_variables: false,
25615 }))
25616 .expect("replacement dataset metadata should resolve")
25617 .expect("replacement dataset metadata should exist");
25618 assert!(!metadata_without_details.variables_included);
25619 assert_eq!(
25620 metadata_without_details.variable_count,
25621 replacement.dataset.variables.len()
25622 );
25623 assert!(metadata_without_details.variables.is_empty());
25624
25625 let metadata_with_details = block_on(service.get_study_dataset(GetStudyDatasetRequest {
25626 study_id: study.study_id,
25627 dataset_name: first.dataset.catalog.asset.name.clone(),
25628 with_variables: true,
25629 }))
25630 .expect("replacement dataset metadata with variables should resolve")
25631 .expect("replacement dataset metadata with variables should exist");
25632 assert!(metadata_with_details.variables_included);
25633 assert_eq!(
25634 metadata_with_details.variable_count,
25635 replacement.dataset.variables.len()
25636 );
25637 assert_eq!(
25638 metadata_with_details.variables.len(),
25639 replacement.dataset.variables.len()
25640 );
25641
25642 let _ = fs::remove_dir_all(&lake_root);
25643 }
25644
25645 #[test]
25646 fn datafile_to_dataset_validates_explicit_variables_against_loaded_csv_columns() {
25647 let repositories = AssetWorkflowRepositories::new();
25648 let service = repositories.service();
25649 let study = test_study("p1_13_explicit_csv_validation_study");
25650 repositories.seed_study(study.clone());
25651 let domain = test_domain(90_303, "p1_13_explicit_csv_validation_domain");
25652 repositories.seed_study_domain(study.study_id, domain.clone());
25653 let source_path = workspace_file("data/ingest.csv");
25654 let lake_root = temp_lake_root_path();
25655 let ingested = block_on(service.ingest_file(IngestFileRequest {
25656 classification: ahri_tre_types::IngestClassification::derived_high(),
25657 study_id: study.study_id,
25658 asset_id: None,
25659 asset_name: nc_name("p1_13_explicit_csv_validation_file"),
25660 version_id: None,
25661 source: IngestFileSource::LocalPath {
25662 path: source_path.display().to_string(),
25663 },
25664 lake_root: lake_root.display().to_string(),
25665 edam_format: Some("EDAM:format_3752".to_string()),
25666 compress: true,
25667 encrypt: None,
25668 description: Some("Managed CSV datafile".to_string()),
25669 agent_instructions: None,
25670 version_note: Some("Initial managed datafile".to_string()),
25671 transformation: test_transformation(
25672 101,
25673 TransformationType::Ingest,
25674 "ingest csv for explicit metadata validation",
25675 ),
25676 }))
25677 .expect("source datafile ingest should succeed");
25678 let mut session = test_lake_session(&lake_root);
25679
25680 let error = block_on(service.datafile_to_dataset(
25681 &mut session,
25682 DataFileToDatasetRequest {
25683 study_id: study.study_id,
25684 metadata_domain_id: None,
25685 dataset_asset_id: None,
25686 dataset_name: nc_name("p1_13_explicit_csv_validation_dataset"),
25687 dataset_version_id: None,
25688 source_datafile_id: ingested.datafile.datafile_id,
25689 input_format: Some("csv".to_string()),
25690 header: None,
25691 delimiter: None,
25692 null_strings: Vec::new(),
25693 json_format: None,
25694 sheet: None,
25695 description: Some("Dataset loaded with explicit metadata".to_string()),
25696 agent_instructions: None,
25697 version_note: Some("Initial datafile-derived dataset".to_string()),
25698 transformation: test_transformation(
25699 102,
25700 TransformationType::Transform,
25701 "transform datafile with invalid explicit metadata",
25702 ),
25703 strict_metadata: false,
25704 sample_rows: None,
25705 vocabulary_threshold: None,
25706 force_metadata_updates: false,
25707 variable_registrations: vec![DatasetVariableRegistrationRequest {
25708 variable: test_variable(
25709 10_303,
25710 domain.domain_id.0,
25711 "missing_column",
25712 KeyRole::None,
25713 None,
25714 ),
25715 row_role: None,
25716 vocabulary: None,
25717 vocabulary_items: Vec::new(),
25718 }],
25719 },
25720 ))
25721 .expect_err("explicit variable should still be validated against CSV columns");
25722
25723 assert!(
25724 error
25725 .to_string()
25726 .contains("dataset column not found for variable missing_column"),
25727 "{error}"
25728 );
25729
25730 let _ = fs::remove_dir_all(&lake_root);
25731 }
25732
25733 #[test]
25734 fn datafile_to_dataset_auto_sanitizes_and_deduplicates_variable_names() {
25735 let repositories = AssetWorkflowRepositories::new();
25736 let service = repositories.service();
25737 let study = test_study("p1_13_auto_sanitize_study");
25738 repositories.seed_study(study.clone());
25739 let domain = test_domain(90_304, "p1_13_auto_sanitize_domain");
25740 repositories.seed_study_domain(study.study_id, domain.clone());
25741 let columns = vec![
25742 ImportedColumn {
25743 name: "First Name".to_string(),
25744 duckdb_type: "VARCHAR".to_string(),
25745 },
25746 ImportedColumn {
25747 name: "first/name".to_string(),
25748 duckdb_type: "VARCHAR".to_string(),
25749 },
25750 ImportedColumn {
25751 name: "123score".to_string(),
25752 duckdb_type: "INTEGER".to_string(),
25753 },
25754 ];
25755
25756 let (registrations, warnings) =
25757 block_on(service.automatic_column_shape_variable_registrations(
25758 study.study_id,
25759 None,
25760 DatasetFileFormat::Csv,
25761 &columns,
25762 None,
25763 &BTreeMap::new(),
25764 ))
25765 .expect("automatic registration should sanitize source columns");
25766
25767 let names = registrations
25768 .iter()
25769 .map(|registration| match registration {
25770 DatasetVariableRegistrationInput::New(registration) => {
25771 registration.variable.name.as_str()
25772 }
25773 DatasetVariableRegistrationInput::Explicit(_) => {
25774 panic!("automatic registrations should be new variables")
25775 }
25776 })
25777 .collect::<Vec<_>>();
25778 assert_eq!(names, vec!["first_name", "first_name_2", "column_123score"]);
25779 assert!(names.iter().all(|name| NcName::parse(*name).is_ok()));
25780 assert!(
25781 warnings
25782 .iter()
25783 .any(|warning| warning.contains("column First Name")
25784 && warning.contains("using TRE variable first_name")),
25785 "{warnings:?}"
25786 );
25787 assert!(
25788 warnings
25789 .iter()
25790 .any(|warning| warning.contains("column first/name")
25791 && warning.contains("using TRE variable first_name_2")),
25792 "{warnings:?}"
25793 );
25794 assert!(
25795 warnings
25796 .iter()
25797 .any(|warning| warning.contains("column 123score")
25798 && warning.contains("using TRE variable column_123score")),
25799 "{warnings:?}"
25800 );
25801 }
25802
25803 #[test]
25804 fn missing_dataset_variable_error_lists_registered_names() {
25805 let variables = ["individual_id", "household_id"]
25806 .into_iter()
25807 .enumerate()
25808 .map(|(index, name)| {
25809 let variable =
25810 test_variable(1000 + index as i64, 90_304, name, KeyRole::External, None);
25811 DatasetVariableMetadata {
25812 link: ahri_tre_types::DatasetVariableLinkRecord {
25813 dataset_id: VersionId(uuid::Uuid::from_u128(1)),
25814 variable_id: variable.variable_id,
25815 row_role: variable.key_role,
25816 },
25817 variable,
25818 value_type: ahri_tre_types::ValueTypeRecord {
25819 value_type_id: ValueTypeId(3),
25820 value_type: "string".to_string(),
25821 description: None,
25822 },
25823 vocabulary: None,
25824 vocabulary_items: Vec::new(),
25825 }
25826 })
25827 .collect::<Vec<_>>();
25828
25829 let error = require_dataset_variable_id(&variables, "IndividualId")
25830 .expect_err("CamelCase source column should not match registered variable name");
25831
25832 assert!(
25833 error.to_string().contains(
25834 "dataset variable not found: IndividualId; available variables: individual_id, household_id"
25835 ),
25836 "{error}"
25837 );
25838 }
25839
25840 #[test]
25841 fn datafile_to_dataset_validates_sanitized_auto_variables_against_source_columns() {
25842 let repositories = AssetWorkflowRepositories::new();
25843 let service = repositories.service();
25844 let study = test_study("p1_13_auto_sanitized_validation_study");
25845 repositories.seed_study(study.clone());
25846 let domain = test_domain(90_305, "p1_13_auto_sanitized_validation_domain");
25847 repositories.seed_study_domain(study.study_id, domain.clone());
25848 let columns = vec![
25849 ImportedColumn {
25850 name: "First Name".to_string(),
25851 duckdb_type: "VARCHAR".to_string(),
25852 },
25853 ImportedColumn {
25854 name: "123score".to_string(),
25855 duckdb_type: "INTEGER".to_string(),
25856 },
25857 ];
25858 let loaded_column_names = columns
25859 .iter()
25860 .map(|column| column.name.clone())
25861 .collect::<Vec<_>>();
25862 let (registrations, _warnings) =
25863 block_on(service.automatic_column_shape_variable_registrations(
25864 study.study_id,
25865 None,
25866 DatasetFileFormat::Csv,
25867 &columns,
25868 None,
25869 &BTreeMap::new(),
25870 ))
25871 .expect("automatic registration should sanitize source columns");
25872
25873 let registered = block_on(service.register_dataset_variables_internal(
25874 VersionId(uuid::Uuid::from_u128(90_305)),
25875 Some(domain.domain_id),
25876 registrations,
25877 Some(&loaded_column_names),
25878 false,
25879 ))
25880 .expect("sanitized automatic variables should validate against original loaded columns");
25881
25882 assert_eq!(
25883 registered
25884 .iter()
25885 .map(|registered| registered.variable.name.as_str())
25886 .collect::<Vec<_>>(),
25887 vec!["first_name", "column_123score"]
25888 );
25889 }
25890
25891 #[test]
25892 fn datafile_to_dataset_auto_registers_ordinary_json_columns_from_imported_shape() {
25893 let materialized = materialize_json_datafile_with_no_registrations(
25894 "p1_13_json_fallback",
25895 "records.json",
25896 r#"[{"participant_id":"p001","age":34,"score":12.5},{"participant_id":"p002","age":41,"score":9.0}]"#,
25897 "application/json",
25898 Some("array"),
25899 120,
25900 );
25901
25902 assert_eq!(materialized.row_count, 2);
25903 assert_eq!(materialized.column_count, 3);
25904 assert_eq!(materialized.variables.len(), 3);
25905 assert_eq!(materialized.metadata_warnings.len(), 3);
25906 assert!(
25907 materialized
25908 .metadata_warnings
25909 .iter()
25910 .any(|warning| warning.contains(
25911 "automatically registered JSON column participant_id as enumeration from DuckDB imported column shape type"
25912 ))
25913 );
25914 assert!(materialized.metadata_warnings.iter().any(|warning| {
25915 warning.contains("bounded distinct-value profiling inferred governed vocabulary")
25916 }));
25917 let participant_id = materialized
25918 .variables
25919 .iter()
25920 .find(|variable| variable.variable.name.as_str() == "participant_id")
25921 .expect("participant_id variable should register");
25922 assert_eq!(participant_id.variable.value_type_id, ValueTypeId(7));
25923 assert_eq!(
25924 participant_id
25925 .vocabulary_items
25926 .iter()
25927 .map(|item| item.code.as_str())
25928 .collect::<Vec<_>>(),
25929 vec!["p001", "p002"]
25930 );
25931 assert_imported_shape_variable(
25932 &materialized,
25933 "age",
25934 ValueTypeId(1),
25935 "JSON fallback should map DuckDB integer shape to integer",
25936 );
25937 assert_imported_shape_variable(
25938 &materialized,
25939 "score",
25940 ValueTypeId(2),
25941 "JSON fallback should map DuckDB floating shape to decimal",
25942 );
25943 }
25944
25945 #[test]
25946 fn datafile_to_dataset_auto_registers_xlsx_columns_from_imported_shape_when_available() {
25947 let repositories = AssetWorkflowRepositories::new();
25948 let service = repositories.service();
25949 let study = test_study("p1_13_xlsx_fallback_study");
25950 repositories.seed_study(study.clone());
25951 let domain = test_domain(90_640, "p1_13_xlsx_fallback_domain");
25952 repositories.seed_study_domain(study.study_id, domain);
25953 let lake_root = temp_lake_root_path();
25954 fs::create_dir_all(&lake_root).expect("XLSX fallback lake root should be created");
25955 let source_path = lake_root.join("records.xlsx");
25956 if let Err(error) = write_minimal_xlsx_fixture(&source_path) {
25957 eprintln!("skipping XLSX fallback test; XLSX fixture writer unavailable: {error}");
25958 let _ = fs::remove_dir_all(&lake_root);
25959 return;
25960 }
25961 let ingested = block_on(service.ingest_file(IngestFileRequest {
25962 classification: ahri_tre_types::IngestClassification::derived_high(),
25963 study_id: study.study_id,
25964 asset_id: None,
25965 asset_name: nc_name("p1_13_xlsx_fallback_datafile"),
25966 version_id: None,
25967 source: IngestFileSource::LocalPath {
25968 path: source_path.display().to_string(),
25969 },
25970 lake_root: lake_root.display().to_string(),
25971 edam_format: Some(
25972 "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet".to_string(),
25973 ),
25974 compress: true,
25975 encrypt: None,
25976 description: Some("Managed XLSX datafile".to_string()),
25977 agent_instructions: None,
25978 version_note: Some("Initial managed XLSX datafile".to_string()),
25979 transformation: test_transformation(
25980 140,
25981 TransformationType::Ingest,
25982 "ingest XLSX as datafile",
25983 ),
25984 }))
25985 .expect("XLSX source datafile ingest should succeed");
25986 let mut session = test_lake_session(&lake_root);
25987
25988 let materialized = block_on(service.datafile_to_dataset(
25989 &mut session,
25990 DataFileToDatasetRequest {
25991 study_id: study.study_id,
25992 metadata_domain_id: None,
25993 dataset_asset_id: None,
25994 dataset_name: nc_name("p1_13_xlsx_fallback_dataset"),
25995 dataset_version_id: None,
25996 source_datafile_id: ingested.datafile.datafile_id,
25997 input_format: Some(
25998 "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet".to_string(),
25999 ),
26000 header: None,
26001 delimiter: None,
26002 null_strings: Vec::new(),
26003 json_format: None,
26004 sheet: None,
26005 description: Some("Dataset loaded from a managed XLSX datafile".to_string()),
26006 agent_instructions: None,
26007 version_note: Some("Initial XLSX datafile-derived dataset".to_string()),
26008 transformation: test_transformation(
26009 141,
26010 TransformationType::Transform,
26011 "transform XLSX datafile to dataset",
26012 ),
26013 strict_metadata: false,
26014 sample_rows: None,
26015 vocabulary_threshold: None,
26016 force_metadata_updates: false,
26017 variable_registrations: Vec::new(),
26018 },
26019 ));
26020
26021 let materialized = match materialized {
26022 Ok(materialized) => materialized,
26023 Err(error) if xlsx_reader_unavailable(&error) => {
26024 eprintln!("skipping XLSX fallback test; DuckDB XLSX reader unavailable: {error}");
26025 let _ = fs::remove_dir_all(&lake_root);
26026 return;
26027 }
26028 Err(error) => panic!("XLSX datafile-to-dataset should succeed: {error}"),
26029 };
26030
26031 assert_eq!(materialized.row_count, 2);
26032 assert_eq!(materialized.column_count, 3);
26033 assert_eq!(materialized.variables.len(), 3);
26034 assert_eq!(materialized.metadata_warnings.len(), 3);
26035 assert!(materialized.metadata_warnings.iter().all(|warning| {
26036 warning.contains("automatically registered XLSX column")
26037 && warning.contains("DuckDB imported column shape type")
26038 }));
26039 assert_imported_shape_variable(
26040 &materialized,
26041 "year",
26042 ValueTypeId(2),
26043 "XLSX fallback should map DuckDB numeric shape to float",
26044 );
26045 assert_imported_shape_variable(
26046 &materialized,
26047 "population",
26048 ValueTypeId(2),
26049 "XLSX fallback should map DuckDB numeric shape to float",
26050 );
26051 let country = materialized
26052 .variables
26053 .iter()
26054 .find(|variable| variable.variable.name.as_str() == "country")
26055 .expect("country variable should register");
26056 assert_eq!(country.variable.value_type_id, ValueTypeId(7));
26057 assert!(
26058 materialized.metadata_warnings.iter().any(|warning| warning
26059 .contains("bounded distinct-value profiling inferred governed vocabulary")),
26060 "{:?}",
26061 materialized.metadata_warnings
26062 );
26063
26064 let _ = fs::remove_dir_all(&lake_root);
26065 }
26066
26067 #[test]
26068 fn datafile_to_dataset_auto_registers_ndjson_columns_from_imported_shape() {
26069 let materialized = materialize_json_datafile_with_no_registrations(
26070 "p1_13_ndjson_fallback",
26071 "records.ndjson",
26072 "{\"participant_id\":\"p001\",\"visit_count\":2}\n{\"participant_id\":\"p002\",\"visit_count\":5}\n",
26073 "application/x-ndjson",
26074 Some("newline_delimited"),
26075 130,
26076 );
26077
26078 assert_eq!(materialized.row_count, 2);
26079 assert_eq!(materialized.column_count, 2);
26080 assert_eq!(materialized.variables.len(), 2);
26081 assert_eq!(materialized.metadata_warnings.len(), 2);
26082 assert!(
26083 materialized
26084 .metadata_warnings
26085 .iter()
26086 .all(|warning| { warning.contains("automatically registered JSON column") })
26087 );
26088 let participant_id = materialized
26089 .variables
26090 .iter()
26091 .find(|variable| variable.variable.name.as_str() == "participant_id")
26092 .expect("participant_id variable should register");
26093 assert_eq!(participant_id.variable.value_type_id, ValueTypeId(7));
26094 assert_eq!(
26095 participant_id
26096 .vocabulary_items
26097 .iter()
26098 .map(|item| item.code.as_str())
26099 .collect::<Vec<_>>(),
26100 vec!["p001", "p002"]
26101 );
26102 assert_imported_shape_variable(
26103 &materialized,
26104 "visit_count",
26105 ValueTypeId(1),
26106 "NDJSON fallback should map DuckDB integer shape to integer",
26107 );
26108 }
26109
26110 #[test]
26111 fn datafile_to_dataset_auto_registration_applies_json_metadata_contract_proposals() {
26112 let repositories = AssetWorkflowRepositories::new();
26113 let service = repositories.service();
26114 let study = test_study("p1_13_json_metadata_contract_study");
26115 repositories.seed_study(study.clone());
26116 let domain = test_domain(90_660, "p1_13_json_metadata_contract_domain");
26117 repositories.seed_study_domain(study.study_id, domain);
26118 let lake_root = temp_lake_root_path();
26119 fs::create_dir_all(&lake_root).expect("JSON metadata contract lake root should exist");
26120 let source_path = lake_root.join("schema.json");
26121 fs::write(
26122 &source_path,
26123 r#"{
26124 "properties": {
26125 "age": {"type": "integer", "description": "Age in years"},
26126 "status": {
26127 "type": "string",
26128 "description": "Common status",
26129 "enum": ["A", "D"],
26130 "enumNames": ["Active", "Dormant"]
26131 }
26132 },
26133 "ahri_tre": {
26134 "fields": [{
26135 "field_name": "status",
26136 "variable_name": "status_code",
26137 "description": "AHRI TRE status",
26138 "vocabulary_name": "status_vocab",
26139 "vocabulary_items": {"A": "Alive", "D": "Dead"}
26140 }]
26141 }
26142 }"#,
26143 )
26144 .expect("JSON metadata contract fixture should write");
26145 let (proposals, warnings) =
26146 AppService::field_metadata_proposals(DatasetFileFormat::Json, &source_path)
26147 .expect("JSON metadata contract should parse");
26148 assert!(warnings.is_empty());
26149 let columns = vec![
26150 ImportedColumn {
26151 name: "status".to_string(),
26152 duckdb_type: "VARCHAR".to_string(),
26153 },
26154 ImportedColumn {
26155 name: "age".to_string(),
26156 duckdb_type: "BIGINT".to_string(),
26157 },
26158 ];
26159
26160 let (registrations, metadata_warnings) =
26161 block_on(service.automatic_column_shape_variable_registrations(
26162 study.study_id,
26163 None,
26164 DatasetFileFormat::Json,
26165 &columns,
26166 Some(&proposals),
26167 &BTreeMap::new(),
26168 ))
26169 .expect("JSON metadata proposals should feed automatic registration");
26170
26171 assert_eq!(registrations.len(), 2);
26172 let status = registrations
26173 .iter()
26174 .find_map(|registration| match registration {
26175 DatasetVariableRegistrationInput::New(registration)
26176 if registration.variable.name.as_str() == "status_code" =>
26177 {
26178 Some(registration)
26179 }
26180 _ => None,
26181 })
26182 .expect("AHRI_TRE JSON variable_name should be applied");
26183 assert_eq!(status.variable.value_type_id, ValueTypeId(7));
26184 assert_eq!(
26185 status.variable.description.as_deref(),
26186 Some("AHRI TRE status")
26187 );
26188 assert_eq!(
26189 status
26190 .vocabulary
26191 .as_ref()
26192 .map(|vocabulary| vocabulary.name.as_str()),
26193 Some("status_vocab")
26194 );
26195 assert_eq!(
26196 status
26197 .vocabulary_items
26198 .iter()
26199 .map(|item| (item.value, item.code.as_str(), item.description.as_deref()))
26200 .collect::<Vec<_>>(),
26201 vec![(1, "A", Some("Alive")), (2, "D", Some("Dead"))]
26202 );
26203 assert!(
26204 metadata_warnings
26205 .iter()
26206 .any(|warning| warning.contains("rich field metadata applied")),
26207 "{metadata_warnings:?}"
26208 );
26209
26210 let _ = fs::remove_dir_all(&lake_root);
26211 }
26212
26213 #[test]
26214 fn json_metadata_contract_ambiguous_enum_warning_is_reported_without_vocabulary() {
26215 let lake_root = temp_lake_root_path();
26216 fs::create_dir_all(&lake_root).expect("ambiguous JSON metadata lake root should exist");
26217 let source_path = lake_root.join("schema.json");
26218 fs::write(
26219 &source_path,
26220 r#"{"properties":{"status":{"type":"string","enum":["A","D"]}}}"#,
26221 )
26222 .expect("ambiguous JSON metadata fixture should write");
26223
26224 let (proposals, warnings) =
26225 AppService::field_metadata_proposals(DatasetFileFormat::Json, &source_path)
26226 .expect("ambiguous JSON metadata should parse as warnings");
26227
26228 assert!(warnings.is_empty());
26229 assert_eq!(proposals.len(), 1);
26230 assert!(proposals[0].vocabulary_items.is_empty());
26231 assert!(
26232 proposals[0]
26233 .warnings
26234 .iter()
26235 .any(|warning| warning.contains("expected enumNames or oneOf const labels")),
26236 "{:?}",
26237 proposals[0].warnings
26238 );
26239
26240 let _ = fs::remove_dir_all(&lake_root);
26241 }
26242
26243 #[test]
26244 fn automatic_metadata_rejects_unsupported_boolean_value_type_name() {
26245 let repositories = AssetWorkflowRepositories::new();
26246 let service = repositories.service();
26247 let study = test_study("p1_18_value_type_boolean_metadata_study");
26248 repositories.seed_study(study.clone());
26249 let domain = test_domain(90_662, "p1_18_value_type_boolean_metadata_domain");
26250 repositories.seed_study_domain(study.study_id, domain);
26251 let columns = vec![ImportedColumn {
26252 name: "consented".to_string(),
26253 duckdb_type: "BOOLEAN".to_string(),
26254 }];
26255 let proposals = vec![FieldMetadataProposal {
26256 field_name: "consented".to_string(),
26257 variable_name: None,
26258 value_type_id: None,
26259 value_type: Some("boolean".to_string()),
26260 description: None,
26261 vocabulary_name: None,
26262 vocabulary_description: None,
26263 vocabulary_items: Vec::new(),
26264 warnings: Vec::new(),
26265 }];
26266
26267 let error = block_on(service.automatic_column_shape_variable_registrations(
26268 study.study_id,
26269 None,
26270 DatasetFileFormat::Json,
26271 &columns,
26272 Some(&proposals),
26273 &BTreeMap::new(),
26274 ))
26275 .expect_err("explicit unsupported value type metadata should fail");
26276
26277 assert!(
26278 error
26279 .to_string()
26280 .contains("unsupported TRE value_type metadata boolean"),
26281 "{error}"
26282 );
26283 }
26284
26285 #[test]
26286 fn automatic_metadata_accepts_multiresponse_value_type_name() {
26287 let repositories = AssetWorkflowRepositories::new();
26288 let service = repositories.service();
26289 let study = test_study("p1_18_value_type_multiresponse_metadata_study");
26290 repositories.seed_study(study.clone());
26291 let domain = test_domain(90_663, "p1_18_value_type_multiresponse_metadata_domain");
26292 repositories.seed_study_domain(study.study_id, domain);
26293 let columns = vec![ImportedColumn {
26294 name: "selected_codes".to_string(),
26295 duckdb_type: "VARCHAR".to_string(),
26296 }];
26297 let proposals = vec![FieldMetadataProposal {
26298 field_name: "selected_codes".to_string(),
26299 variable_name: None,
26300 value_type_id: None,
26301 value_type: Some("multiresponse".to_string()),
26302 description: None,
26303 vocabulary_name: None,
26304 vocabulary_description: None,
26305 vocabulary_items: Vec::new(),
26306 warnings: Vec::new(),
26307 }];
26308
26309 let (registrations, _warnings) =
26310 block_on(service.automatic_column_shape_variable_registrations(
26311 study.study_id,
26312 None,
26313 DatasetFileFormat::Json,
26314 &columns,
26315 Some(&proposals),
26316 &BTreeMap::new(),
26317 ))
26318 .expect("multiresponse metadata should be supported");
26319
26320 let registration = registrations
26321 .iter()
26322 .find_map(|registration| match registration {
26323 DatasetVariableRegistrationInput::New(registration) => Some(registration),
26324 DatasetVariableRegistrationInput::Explicit(_) => None,
26325 })
26326 .expect("automatic registration should create a variable");
26327 assert_eq!(registration.variable.name.as_str(), "selected_codes");
26328 assert_eq!(registration.variable.value_type_id, ValueTypeId(8));
26329 }
26330
26331 #[test]
26332 fn datafile_to_dataset_enriches_parquet_variables_and_vocabularies_from_metadata() {
26333 let repositories = AssetWorkflowRepositories::new();
26334 let service = repositories.service();
26335 let study = test_study("p1_13_parquet_metadata_study");
26336 repositories.seed_study(study.clone());
26337 let domain = test_domain(90_641, "p1_13_parquet_metadata_domain");
26338 repositories.seed_study_domain(study.study_id, domain);
26339 let lake_root = temp_lake_root_path();
26340 fs::create_dir_all(&lake_root).expect("Parquet metadata lake root should be created");
26341 let source_path = lake_root.join("metadata_source.parquet");
26342 write_parquet_metadata_fixture(&source_path);
26343 let ingested = block_on(service.ingest_file(IngestFileRequest {
26344 classification: ahri_tre_types::IngestClassification::derived_high(),
26345 study_id: study.study_id,
26346 asset_id: None,
26347 asset_name: nc_name("p1_13_parquet_metadata_datafile"),
26348 version_id: None,
26349 source: IngestFileSource::LocalPath {
26350 path: source_path.display().to_string(),
26351 },
26352 lake_root: lake_root.display().to_string(),
26353 edam_format: Some("parquet".to_string()),
26354 compress: true,
26355 encrypt: None,
26356 description: Some("Managed Parquet datafile".to_string()),
26357 agent_instructions: None,
26358 version_note: Some("Initial managed Parquet datafile".to_string()),
26359 transformation: test_transformation(142, TransformationType::Ingest, "ingest Parquet"),
26360 }))
26361 .expect("Parquet source datafile ingest should succeed");
26362 let mut session = test_lake_session(&lake_root);
26363
26364 let materialized = block_on(service.datafile_to_dataset(
26365 &mut session,
26366 DataFileToDatasetRequest {
26367 study_id: study.study_id,
26368 metadata_domain_id: None,
26369 dataset_asset_id: None,
26370 dataset_name: nc_name("p1_13_parquet_metadata_dataset"),
26371 dataset_version_id: None,
26372 source_datafile_id: ingested.datafile.datafile_id,
26373 input_format: Some("parquet".to_string()),
26374 header: None,
26375 delimiter: None,
26376 null_strings: Vec::new(),
26377 json_format: None,
26378 sheet: None,
26379 description: Some("Dataset loaded from managed Parquet".to_string()),
26380 agent_instructions: None,
26381 version_note: Some("Initial Parquet datafile-derived dataset".to_string()),
26382 transformation: test_transformation(
26383 143,
26384 TransformationType::Transform,
26385 "transform Parquet datafile to dataset",
26386 ),
26387 strict_metadata: false,
26388 sample_rows: None,
26389 vocabulary_threshold: None,
26390 force_metadata_updates: false,
26391 variable_registrations: Vec::new(),
26392 },
26393 ))
26394 .expect("Parquet datafile-to-dataset should enrich automatic registrations");
26395
26396 assert_eq!(materialized.row_count, 2);
26397 assert_eq!(materialized.column_count, 3);
26398 assert_eq!(materialized.variables.len(), 3);
26399 assert_imported_shape_variable(
26400 &materialized,
26401 "age",
26402 ValueTypeId(1),
26403 "unannotated Parquet field should fall back to DuckDB imported shape",
26404 );
26405 let status = materialized
26406 .variables
26407 .iter()
26408 .find(|variable| variable.variable.name.as_str() == "status_code")
26409 .expect("AHRI_TRE variable_name should be used");
26410 assert_eq!(status.variable.value_type_id, ValueTypeId(7));
26411 assert_eq!(
26412 status.variable.description.as_deref(),
26413 Some("AHRI TRE status description")
26414 );
26415 assert_eq!(
26416 status
26417 .vocabulary
26418 .as_ref()
26419 .map(|vocabulary| vocabulary.name.as_str()),
26420 Some("status_vocab")
26421 );
26422 assert_eq!(
26423 status
26424 .vocabulary_items
26425 .iter()
26426 .map(|item| (item.value, item.code.as_str(), item.description.as_deref()))
26427 .collect::<Vec<_>>(),
26428 vec![(1, "A", Some("Alive")), (2, "D", Some("Dead"))]
26429 );
26430 let sex = materialized
26431 .variables
26432 .iter()
26433 .find(|variable| variable.variable.name.as_str() == "sex_code")
26434 .expect("sanitized conventional categorical field should register");
26435 assert_eq!(sex.variable.value_type_id, ValueTypeId(7));
26436 assert_eq!(sex.variable.description.as_deref(), Some("Sex at baseline"));
26437 assert_eq!(
26438 sex.vocabulary_items
26439 .iter()
26440 .map(|item| (item.value, item.code.as_str(), item.description.as_deref()))
26441 .collect::<Vec<_>>(),
26442 vec![(1, "F", Some("Female")), (2, "M", Some("Male"))]
26443 );
26444 assert!(
26445 materialized
26446 .metadata_warnings
26447 .iter()
26448 .any(|warning| warning.contains("rich Parquet field metadata applied")),
26449 "{:?}",
26450 materialized.metadata_warnings
26451 );
26452
26453 let _ = fs::remove_dir_all(&lake_root);
26454 }
26455
26456 #[test]
26457 fn datafile_to_dataset_enriches_arrow_ipc_variables_and_vocabularies_from_metadata() {
26458 let repositories = AssetWorkflowRepositories::new();
26459 let service = repositories.service();
26460 let study = test_study("p1_13_arrow_ipc_metadata_study");
26461 repositories.seed_study(study.clone());
26462 let domain = test_domain(90_643, "p1_13_arrow_ipc_metadata_domain");
26463 repositories.seed_study_domain(study.study_id, domain);
26464 let lake_root = temp_lake_root_path();
26465 fs::create_dir_all(&lake_root).expect("Arrow IPC metadata lake root should be created");
26466 let source_path = lake_root.join("metadata_source.arrow");
26467 write_arrow_ipc_metadata_fixture(&source_path);
26468 let ingested = block_on(service.ingest_file(IngestFileRequest {
26469 classification: ahri_tre_types::IngestClassification::derived_high(),
26470 study_id: study.study_id,
26471 asset_id: None,
26472 asset_name: nc_name("p1_13_arrow_ipc_metadata_datafile"),
26473 version_id: None,
26474 source: IngestFileSource::LocalPath {
26475 path: source_path.display().to_string(),
26476 },
26477 lake_root: lake_root.display().to_string(),
26478 edam_format: Some("application/vnd.apache.arrow.file".to_string()),
26479 compress: true,
26480 encrypt: None,
26481 description: Some("Managed Arrow IPC datafile".to_string()),
26482 agent_instructions: None,
26483 version_note: Some("Initial managed Arrow IPC datafile".to_string()),
26484 transformation: test_transformation(
26485 146,
26486 TransformationType::Ingest,
26487 "ingest Arrow IPC",
26488 ),
26489 }))
26490 .expect("Arrow IPC source datafile ingest should succeed");
26491 let mut session = test_lake_session(&lake_root);
26492
26493 let materialized = block_on(service.datafile_to_dataset(
26494 &mut session,
26495 DataFileToDatasetRequest {
26496 study_id: study.study_id,
26497 metadata_domain_id: None,
26498 dataset_asset_id: None,
26499 dataset_name: nc_name("p1_13_arrow_ipc_metadata_dataset"),
26500 dataset_version_id: None,
26501 source_datafile_id: ingested.datafile.datafile_id,
26502 input_format: Some("application/vnd.apache.arrow.file".to_string()),
26503 header: None,
26504 delimiter: None,
26505 null_strings: Vec::new(),
26506 json_format: None,
26507 sheet: None,
26508 description: Some("Dataset loaded from managed Arrow IPC".to_string()),
26509 agent_instructions: None,
26510 version_note: Some("Initial Arrow IPC datafile-derived dataset".to_string()),
26511 transformation: test_transformation(
26512 147,
26513 TransformationType::Transform,
26514 "transform Arrow IPC datafile to dataset",
26515 ),
26516 strict_metadata: false,
26517 sample_rows: None,
26518 vocabulary_threshold: None,
26519 force_metadata_updates: false,
26520 variable_registrations: Vec::new(),
26521 },
26522 ))
26523 .expect("Arrow IPC datafile-to-dataset should enrich automatic registrations");
26524
26525 assert_eq!(materialized.row_count, 2);
26526 assert_eq!(materialized.column_count, 3);
26527 assert_eq!(materialized.variables.len(), 3);
26528 assert_imported_shape_variable(
26529 &materialized,
26530 "age",
26531 ValueTypeId(1),
26532 "unannotated Arrow IPC field should fall back to DuckDB imported shape",
26533 );
26534 let status = materialized
26535 .variables
26536 .iter()
26537 .find(|variable| variable.variable.name.as_str() == "status_code")
26538 .expect("AHRI_TRE variable_name should be used");
26539 assert_eq!(status.variable.value_type_id, ValueTypeId(7));
26540 assert_eq!(
26541 status.variable.description.as_deref(),
26542 Some("AHRI TRE status description")
26543 );
26544 assert_eq!(
26545 status
26546 .vocabulary_items
26547 .iter()
26548 .map(|item| (item.value, item.code.as_str(), item.description.as_deref()))
26549 .collect::<Vec<_>>(),
26550 vec![(1, "A", Some("Alive")), (2, "D", Some("Dead"))]
26551 );
26552 assert!(
26553 materialized
26554 .metadata_warnings
26555 .iter()
26556 .any(|warning| warning.contains("rich Arrow IPC field metadata applied")),
26557 "{:?}",
26558 materialized.metadata_warnings
26559 );
26560 assert_eq!(
26561 materialized
26562 .lineage
26563 .inputs
26564 .iter()
26565 .map(|input| input.version_id)
26566 .collect::<Vec<_>>(),
26567 vec![ingested.datafile.datafile_id]
26568 );
26569 assert_eq!(
26570 materialized
26571 .lineage
26572 .outputs
26573 .iter()
26574 .map(|output| output.version_id)
26575 .collect::<Vec<_>>(),
26576 vec![materialized.dataset.dataset_id]
26577 );
26578 assert!(
26579 !lake_root
26580 .join("__tre_duckdb_stage")
26581 .join(format!(
26582 "materialized-{}.arrow",
26583 ingested.datafile.datafile_id.0
26584 ))
26585 .exists()
26586 );
26587
26588 let _ = fs::remove_dir_all(&lake_root);
26589 }
26590
26591 #[test]
26592 fn datafile_to_dataset_preserves_explicit_registrations_over_parquet_metadata() {
26593 let repositories = AssetWorkflowRepositories::new();
26594 let service = repositories.service();
26595 let study = test_study("p1_13_explicit_parquet_metadata_study");
26596 repositories.seed_study(study.clone());
26597 let domain = test_domain(90_642, "p1_13_explicit_parquet_metadata_domain");
26598 repositories.seed_study_domain(study.study_id, domain.clone());
26599 let lake_root = temp_lake_root_path();
26600 fs::create_dir_all(&lake_root).expect("explicit Parquet lake root should be created");
26601 let source_path = lake_root.join("explicit_metadata_source.parquet");
26602 write_parquet_metadata_fixture(&source_path);
26603 let ingested = block_on(service.ingest_file(IngestFileRequest {
26604 classification: ahri_tre_types::IngestClassification::derived_high(),
26605 study_id: study.study_id,
26606 asset_id: None,
26607 asset_name: nc_name("p1_13_explicit_parquet_datafile"),
26608 version_id: None,
26609 source: IngestFileSource::LocalPath {
26610 path: source_path.display().to_string(),
26611 },
26612 lake_root: lake_root.display().to_string(),
26613 edam_format: Some("parquet".to_string()),
26614 compress: true,
26615 encrypt: None,
26616 description: Some("Managed explicit Parquet datafile".to_string()),
26617 agent_instructions: None,
26618 version_note: Some("Initial explicit Parquet datafile".to_string()),
26619 transformation: test_transformation(144, TransformationType::Ingest, "ingest Parquet"),
26620 }))
26621 .expect("Parquet source datafile ingest should succeed");
26622 let mut session = test_lake_session(&lake_root);
26623
26624 let materialized = block_on(service.datafile_to_dataset(
26625 &mut session,
26626 DataFileToDatasetRequest {
26627 study_id: study.study_id,
26628 metadata_domain_id: None,
26629 dataset_asset_id: None,
26630 dataset_name: nc_name("p1_13_explicit_parquet_metadata_dataset"),
26631 dataset_version_id: None,
26632 source_datafile_id: ingested.datafile.datafile_id,
26633 input_format: Some("parquet".to_string()),
26634 header: None,
26635 delimiter: None,
26636 null_strings: Vec::new(),
26637 json_format: None,
26638 sheet: None,
26639 description: Some("Dataset loaded with explicit Parquet metadata".to_string()),
26640 agent_instructions: None,
26641 version_note: Some("Initial explicit Parquet dataset".to_string()),
26642 transformation: test_transformation(
26643 145,
26644 TransformationType::Transform,
26645 "transform Parquet with explicit metadata",
26646 ),
26647 strict_metadata: false,
26648 sample_rows: None,
26649 vocabulary_threshold: None,
26650 force_metadata_updates: false,
26651 variable_registrations: vec![DatasetVariableRegistrationRequest {
26652 variable: test_variable(
26653 10_642,
26654 domain.domain_id.0,
26655 "age",
26656 KeyRole::Record,
26657 None,
26658 ),
26659 row_role: Some(KeyRole::Record),
26660 vocabulary: None,
26661 vocabulary_items: Vec::new(),
26662 }],
26663 },
26664 ))
26665 .expect("explicit Parquet registration should override discovered metadata");
26666
26667 assert_eq!(materialized.variables.len(), 1);
26668 assert_eq!(materialized.variables[0].variable.name.as_str(), "age");
26669 assert_eq!(
26670 materialized.variables[0].variable.description.as_deref(),
26671 Some("Variable age")
26672 );
26673 assert!(materialized.variables[0].vocabulary.is_none());
26674 assert_eq!(materialized.metadata_warnings.len(), 1);
26675 assert!(
26676 materialized.metadata_warnings[0].contains(
26677 "explicit variable registrations supplied; ignored automatic/discovered Parquet metadata for 3 imported columns"
26678 ),
26679 "{:?}",
26680 materialized.metadata_warnings
26681 );
26682
26683 let _ = fs::remove_dir_all(&lake_root);
26684 }
26685
26686 #[test]
26687 fn datafile_to_dataset_profiles_low_cardinality_csv_strings_as_vocabulary() {
26688 let materialized = materialize_csv_datafile_with_profiling(
26689 "issue03_penguins_vocab",
26690 3_000,
26691 "species,island,bill_length\nGentoo,Biscoe,46.1\nAdelie,Torgersen,39.1\nNA,Dream,40.3\nChinstrap,Dream,46.5\nAdelie,,38.7\n",
26692 CsvProfilingOptions {
26693 null_strings: vec!["NA".to_string(), "".to_string()],
26694 sample_rows: None,
26695 vocabulary_threshold: Some(3),
26696 force_metadata_updates: false,
26697 },
26698 |_, _| {},
26699 )
26700 .expect("low-cardinality CSV should materialize with profiled vocabulary");
26701
26702 let species = materialized
26703 .variables
26704 .iter()
26705 .find(|variable| variable.variable.name.as_str() == "species")
26706 .expect("species variable should register");
26707 assert_eq!(species.variable.value_type_id, ValueTypeId(7));
26708 assert_eq!(
26709 species
26710 .vocabulary
26711 .as_ref()
26712 .map(|vocabulary| vocabulary.name.as_str()),
26713 Some("species_vocabulary")
26714 );
26715 assert_eq!(
26716 species
26717 .vocabulary_items
26718 .iter()
26719 .map(|item| (item.value, item.code.as_str(), item.description.as_deref()))
26720 .collect::<Vec<_>>(),
26721 vec![
26722 (1, "Adelie", Some("Adelie")),
26723 (2, "Chinstrap", Some("Chinstrap")),
26724 (3, "Gentoo", Some("Gentoo")),
26725 ]
26726 );
26727 assert!(
26728 materialized.metadata_warnings.iter().any(|warning| warning
26729 .contains("bounded distinct-value profiling inferred governed vocabulary")),
26730 "{:?}",
26731 materialized.metadata_warnings
26732 );
26733 }
26734
26735 #[test]
26736 fn datafile_to_dataset_profiles_threshold_and_sample_row_boundaries() {
26737 let high_cardinality = materialize_csv_datafile_with_profiling(
26738 "issue03_threshold_fallback",
26739 3_010,
26740 "species\nAdelie\nChinstrap\nGentoo\n",
26741 CsvProfilingOptions {
26742 null_strings: Vec::new(),
26743 sample_rows: None,
26744 vocabulary_threshold: Some(2),
26745 force_metadata_updates: false,
26746 },
26747 |_, _| {},
26748 )
26749 .expect("high-cardinality CSV should still materialize");
26750 assert_imported_shape_variable(
26751 &high_cardinality,
26752 "species",
26753 ValueTypeId(3),
26754 "distinct count above threshold should fall back to string",
26755 );
26756
26757 let bounded_sample = materialize_csv_datafile_with_profiling(
26758 "issue03_sample_bound",
26759 3_020,
26760 "species\nAdelie\nGentoo\nChinstrap\n",
26761 CsvProfilingOptions {
26762 null_strings: Vec::new(),
26763 sample_rows: Some(2),
26764 vocabulary_threshold: Some(2),
26765 force_metadata_updates: false,
26766 },
26767 |_, _| {},
26768 )
26769 .expect("sample-bounded CSV should materialize");
26770 let species = bounded_sample
26771 .variables
26772 .iter()
26773 .find(|variable| variable.variable.name.as_str() == "species")
26774 .expect("sampled species variable should register");
26775 assert_eq!(species.variable.value_type_id, ValueTypeId(7));
26776 assert_eq!(
26777 species
26778 .vocabulary_items
26779 .iter()
26780 .map(|item| item.code.as_str())
26781 .collect::<Vec<_>>(),
26782 vec!["Adelie", "Gentoo"]
26783 );
26784 }
26785
26786 #[test]
26787 fn datafile_to_dataset_requires_force_to_correct_existing_profiled_metadata() {
26788 let csv = "species\nAdelie\nGentoo\n";
26789 let error = materialize_csv_datafile_with_profiling(
26790 "issue03_force_rejects",
26791 3_030,
26792 csv,
26793 CsvProfilingOptions {
26794 null_strings: Vec::new(),
26795 sample_rows: None,
26796 vocabulary_threshold: Some(2),
26797 force_metadata_updates: false,
26798 },
26799 |repositories, domain| {
26800 repositories.seed_variable(test_variable(
26801 93_030,
26802 domain.domain_id.0,
26803 "species",
26804 KeyRole::None,
26805 None,
26806 ));
26807 },
26808 )
26809 .expect_err("profiled categorical correction should require force");
26810 assert!(
26811 error
26812 .to_string()
26813 .contains("set force_metadata_updates to update metadata"),
26814 "{error}"
26815 );
26816
26817 let materialized = materialize_csv_datafile_with_profiling(
26818 "issue03_force_updates",
26819 3_040,
26820 csv,
26821 CsvProfilingOptions {
26822 null_strings: Vec::new(),
26823 sample_rows: None,
26824 vocabulary_threshold: Some(2),
26825 force_metadata_updates: true,
26826 },
26827 |repositories, domain| {
26828 repositories.seed_variable(test_variable(
26829 93_040,
26830 domain.domain_id.0,
26831 "species",
26832 KeyRole::None,
26833 None,
26834 ));
26835 },
26836 )
26837 .expect("force should allow compatible unlinked metadata correction");
26838 let species = materialized
26839 .variables
26840 .iter()
26841 .find(|variable| variable.variable.name.as_str() == "species")
26842 .expect("species variable should register after force");
26843 assert_eq!(species.variable.value_type_id, ValueTypeId(7));
26844 assert!(species.variable.vocabulary_id.is_some());
26845 assert_eq!(
26846 species
26847 .vocabulary_items
26848 .iter()
26849 .map(|item| item.code.as_str())
26850 .collect::<Vec<_>>(),
26851 vec!["Adelie", "Gentoo"]
26852 );
26853 assert!(
26854 materialized.metadata_warnings.iter().any(|warning| warning
26855 .contains("force_metadata_updates enabled: automatic metadata will update species value_type_id")),
26856 "{:?}",
26857 materialized.metadata_warnings
26858 );
26859 }
26860
26861 #[test]
26862 fn datafile_to_dataset_auto_fails_existing_value_type_conflict_in_nonstrict_mode() {
26863 let error = materialize_parquet_metadata_datafile(
26864 "p1_13_parquet_value_type_conflict",
26865 150,
26866 false,
26867 |repositories, domain| {
26868 let mut variable = test_variable(
26869 10_650,
26870 domain.domain_id.0,
26871 "status_code",
26872 KeyRole::None,
26873 None,
26874 );
26875 variable.value_type_id = ValueTypeId(3);
26876 repositories.seed_variable(variable);
26877 },
26878 )
26879 .expect_err("automatic Parquet metadata should reject value-type drift");
26880
26881 assert!(
26882 error.to_string().contains("already exists in domain")
26883 && error.to_string().contains("incompatible value type"),
26884 "{error}"
26885 );
26886 }
26887
26888 #[test]
26889 fn datafile_to_dataset_auto_fails_existing_vocabulary_item_conflict_in_nonstrict_mode() {
26890 let error = materialize_parquet_metadata_datafile(
26891 "p1_13_parquet_vocabulary_item_conflict",
26892 160,
26893 false,
26894 |repositories, domain| {
26895 seed_status_code_variable_and_vocabulary(
26896 repositories,
26897 domain,
26898 "AHRI TRE status description",
26899 None,
26900 vec![
26901 VocabularyItemRecord {
26902 vocabulary_item_id: VocabularyItemId(10_660),
26903 vocabulary_id: VocabularyId(10_660),
26904 value: 99,
26905 code: "A".to_string(),
26906 description: Some("Alive".to_string()),
26907 },
26908 VocabularyItemRecord {
26909 vocabulary_item_id: VocabularyItemId(10_661),
26910 vocabulary_id: VocabularyId(10_660),
26911 value: 2,
26912 code: "D".to_string(),
26913 description: Some("Dead".to_string()),
26914 },
26915 ],
26916 );
26917 },
26918 )
26919 .expect_err("automatic Parquet metadata should reject vocabulary item drift");
26920
26921 assert!(
26922 error.to_string().contains("vocabulary item")
26923 && error
26924 .to_string()
26925 .contains("set force_metadata_updates to update metadata"),
26926 "{error}"
26927 );
26928 }
26929
26930 #[test]
26931 fn datafile_to_dataset_auto_warns_for_enrichment_conflicts_in_nonstrict_mode() {
26932 let materialized = materialize_parquet_metadata_datafile(
26933 "p1_13_parquet_enrichment_warning",
26934 170,
26935 false,
26936 |repositories, domain| {
26937 seed_status_code_variable_and_vocabulary(
26938 repositories,
26939 domain,
26940 "Existing status description",
26941 Some("Existing status vocabulary"),
26942 matching_status_vocabulary_items(10_670),
26943 );
26944 },
26945 )
26946 .expect("non-strict automatic Parquet metadata should warn and continue");
26947
26948 let status = materialized
26949 .variables
26950 .iter()
26951 .find(|variable| variable.variable.name.as_str() == "status_code")
26952 .expect("status_code should be reused");
26953 assert_eq!(
26954 status.variable.description.as_deref(),
26955 Some("Existing status description")
26956 );
26957 assert!(
26958 materialized.metadata_warnings.iter().any(|warning| warning
26959 .contains("automatic metadata conflict: status_code existing description")),
26960 "{:?}",
26961 materialized.metadata_warnings
26962 );
26963 assert!(
26964 materialized
26965 .metadata_warnings
26966 .iter()
26967 .any(|warning| warning.contains(
26968 "automatic metadata conflict: status_vocab existing vocabulary description"
26969 )),
26970 "{:?}",
26971 materialized.metadata_warnings
26972 );
26973 }
26974
26975 #[test]
26976 fn datafile_to_dataset_auto_fails_enrichment_conflicts_in_strict_mode() {
26977 let error = materialize_parquet_metadata_datafile(
26978 "p1_13_parquet_strict_enrichment_failure",
26979 180,
26980 true,
26981 |repositories, domain| {
26982 seed_status_code_variable_and_vocabulary(
26983 repositories,
26984 domain,
26985 "Existing status description",
26986 None,
26987 matching_status_vocabulary_items(10_680),
26988 );
26989 },
26990 )
26991 .expect_err("strict automatic Parquet metadata should reject enrichment drift");
26992
26993 assert!(
26994 error
26995 .to_string()
26996 .contains("automatic metadata conflict: status_code existing description"),
26997 "{error}"
26998 );
26999 }
27000
27001 #[test]
27002 fn datafile_to_dataset_arrow_ipc_matches_nonstrict_and_strict_conflict_behavior() {
27003 let materialized = materialize_arrow_ipc_metadata_datafile(
27004 "p1_13_arrow_ipc_enrichment_warning",
27005 190,
27006 false,
27007 |repositories, domain| {
27008 seed_status_code_variable_and_vocabulary(
27009 repositories,
27010 domain,
27011 "Existing status description",
27012 Some("Existing status vocabulary"),
27013 matching_status_vocabulary_items(10_690),
27014 );
27015 },
27016 )
27017 .expect("non-strict automatic Arrow IPC metadata should warn and continue");
27018 assert!(
27019 materialized.metadata_warnings.iter().any(|warning| warning
27020 .contains("automatic metadata conflict: status_code existing description")),
27021 "{:?}",
27022 materialized.metadata_warnings
27023 );
27024
27025 let error = materialize_arrow_ipc_metadata_datafile(
27026 "p1_13_arrow_ipc_strict_enrichment_failure",
27027 200,
27028 true,
27029 |repositories, domain| {
27030 seed_status_code_variable_and_vocabulary(
27031 repositories,
27032 domain,
27033 "Existing status description",
27034 None,
27035 matching_status_vocabulary_items(10_700),
27036 );
27037 },
27038 )
27039 .expect_err("strict automatic Arrow IPC metadata should reject enrichment drift");
27040
27041 assert!(
27042 error
27043 .to_string()
27044 .contains("automatic metadata conflict: status_code existing description"),
27045 "{error}"
27046 );
27047 }
27048
27049 #[test]
27050 fn transform_assets_records_transformation_inputs_and_outputs() {
27051 let repositories = TransformationRepositories::new();
27052 let service = repositories.service();
27053 let transformation =
27054 test_transformation(91, TransformationType::Transform, "harmonize visits");
27055 let input_version_id = VersionId(uuid::Uuid::new_v4());
27056 let output_version_id = VersionId(uuid::Uuid::new_v4());
27057
27058 let lineage = block_on(service.transform_assets(TransformAssetsRequest {
27059 transformation: transformation.clone(),
27060 input_version_ids: vec![input_version_id],
27061 output_version_ids: vec![output_version_id],
27062 }))
27063 .expect("transform_assets should record transformation provenance");
27064
27065 assert_eq!(lineage.transformation, transformation);
27066 assert_eq!(
27067 lineage.inputs,
27068 vec![TransformationInputLinkRecord {
27069 transformation_id: lineage.transformation.transformation_id,
27070 version_id: input_version_id,
27071 }]
27072 );
27073 assert_eq!(
27074 lineage.outputs,
27075 vec![TransformationOutputLinkRecord {
27076 transformation_id: lineage.transformation.transformation_id,
27077 version_id: output_version_id,
27078 }]
27079 );
27080 assert_eq!(
27081 block_on(
27082 repositories
27083 .repository
27084 .list_transformations_producing(output_version_id)
27085 )
27086 .expect("producing transformations should read"),
27087 vec![transformation]
27088 );
27089 }
27090
27091 #[test]
27092 fn list_study_transformations_includes_input_only_semantic_batches() {
27093 let repositories = AssetWorkflowRepositories::new();
27094 let service = repositories.service();
27095 let study = test_study("semantic_transformation_study");
27096 repositories.seed_study(study.clone());
27097
27098 let asset = block_on(
27099 repositories
27100 .assets
27101 .save_asset(&test_asset(study.study_id, "semantic_source")),
27102 )
27103 .expect("asset should save");
27104 let version = block_on(repositories.assets.save_asset_version(&test_asset_version(
27105 asset.asset_id,
27106 1,
27107 0,
27108 0,
27109 "semantic source v1",
27110 )))
27111 .expect("asset version should save");
27112
27113 let materialization =
27114 block_on(service.record_transformation(RecordTransformationRequest {
27115 transformation: test_transformation(
27116 0,
27117 TransformationType::Ingest,
27118 "ingest semantic_source",
27119 ),
27120 input_version_ids: Vec::new(),
27121 output_version_ids: vec![version.version_id],
27122 }))
27123 .expect("output-producing transformation should record");
27124 let semantic_batch = block_on(service.record_transformation(RecordTransformationRequest {
27125 transformation: test_transformation(
27126 0,
27127 TransformationType::Entity,
27128 "ensure participant entity instances from dataset semantic_source",
27129 ),
27130 input_version_ids: vec![version.version_id],
27131 output_version_ids: Vec::new(),
27132 }))
27133 .expect("input-only semantic transformation should record");
27134
27135 let summaries = block_on(service.list_study_transformations(
27136 ListStudyTransformationsRequest {
27137 study: study.name.as_str().to_string(),
27138 },
27139 ))
27140 .expect("study transformations should list");
27141 let descriptions = summaries
27142 .iter()
27143 .map(|summary| summary.transformation.description.as_str())
27144 .collect::<Vec<_>>();
27145
27146 assert_eq!(summaries.len(), 2);
27147 assert!(descriptions.contains(&materialization.transformation.description.as_str()));
27148 assert!(descriptions.contains(&semantic_batch.transformation.description.as_str()));
27149 assert_eq!(
27150 summaries
27151 .iter()
27152 .find(|summary| summary.transformation.transformation_id
27153 == semantic_batch.transformation.transformation_id)
27154 .expect("semantic batch should be listed")
27155 .inputs,
27156 semantic_batch.inputs
27157 );
27158 }
27159
27160 #[test]
27161 fn repeated_transformation_create_intent_gets_distinct_persisted_ids() {
27162 let repositories = TransformationRepositories::new();
27163 let service = repositories.service();
27164 let transformation =
27165 test_transformation(91, TransformationType::Transform, "harmonize visits");
27166 let input_version_id = VersionId(uuid::Uuid::new_v4());
27167
27168 let first = block_on(service.transform_assets(TransformAssetsRequest {
27169 transformation: transformation.clone(),
27170 input_version_ids: vec![input_version_id],
27171 output_version_ids: vec![VersionId(uuid::Uuid::new_v4())],
27172 }))
27173 .expect("first transform should persist provenance");
27174 let second = block_on(service.transform_assets(TransformAssetsRequest {
27175 transformation,
27176 input_version_ids: vec![input_version_id],
27177 output_version_ids: vec![VersionId(uuid::Uuid::new_v4())],
27178 }))
27179 .expect("second transform should persist separate provenance");
27180
27181 assert_ne!(
27182 first.transformation.transformation_id,
27183 second.transformation.transformation_id
27184 );
27185 }
27186
27187 #[test]
27188 fn transformation_source_enrichment_derives_missing_git_fields() {
27189 let transformation = test_transformation_without_source(
27190 95,
27191 TransformationType::Transform,
27192 "derive source provenance",
27193 );
27194 let transformation = NewTransformationRecord {
27195 file_path: Some("/workspace/ahri-tre-rs/crates/ahri_tre_cli/src/app.rs".to_string()),
27196 ..transformation
27197 };
27198
27199 let enriched =
27200 enrich_transformation_source_with_git(&transformation, None, |args| match args {
27201 [
27202 "-C",
27203 "/workspace/ahri-tre-rs/crates/ahri_tre_cli/src",
27204 "rev-parse",
27205 "--show-toplevel",
27206 ] => Some("/workspace/ahri-tre-rs".to_string()),
27207 [
27208 "-C",
27209 "/workspace/ahri-tre-rs",
27210 "config",
27211 "--get",
27212 "remote.origin.url",
27213 ] => Some("git@github.com:AHRIORG/ahri-tre-rs.git".to_string()),
27214 ["-C", "/workspace/ahri-tre-rs", "rev-parse", "HEAD"] => {
27215 Some("abcdef1234567890".to_string())
27216 }
27217 _ => None,
27218 });
27219
27220 assert_eq!(
27221 enriched.repository_url.as_deref(),
27222 Some("https://github.com/AHRIORG/ahri-tre-rs")
27223 );
27224 assert_eq!(enriched.commit_hash.as_deref(), Some("abcdef1"));
27225 assert_eq!(enriched.file_path, transformation.file_path);
27226 }
27227
27228 #[test]
27229 fn transformation_source_enrichment_preserves_explicit_values() {
27230 let transformation = test_transformation(
27231 96,
27232 TransformationType::Transform,
27233 "preserve explicit source provenance",
27234 );
27235
27236 let enriched = enrich_transformation_source_with_git(&transformation, None, |_| {
27237 panic!("explicit source provenance should not invoke git discovery")
27238 });
27239
27240 assert_eq!(enriched, transformation);
27241 }
27242
27243 #[test]
27244 fn transformation_source_enrichment_uses_caller_file_when_path_is_missing() {
27245 let transformation = test_transformation_without_source(
27246 99,
27247 TransformationType::Transform,
27248 "derive caller source provenance",
27249 );
27250
27251 let enriched = enrich_transformation_source_with_git(
27252 &transformation,
27253 Some("/workspace/ahri-tre-rs/crates/ahri_tre_cli/src/app.rs"),
27254 |args| match args {
27255 [
27256 "-C",
27257 "/workspace/ahri-tre-rs/crates/ahri_tre_cli/src",
27258 "rev-parse",
27259 "--show-toplevel",
27260 ] => Some("/workspace/ahri-tre-rs".to_string()),
27261 [
27262 "-C",
27263 "/workspace/ahri-tre-rs",
27264 "config",
27265 "--get",
27266 "remote.origin.url",
27267 ] => Some("https://github.com/AHRIORG/ahri-tre-rs.git".to_string()),
27268 ["-C", "/workspace/ahri-tre-rs", "rev-parse", "HEAD"] => {
27269 Some("abcdef1234567890".to_string())
27270 }
27271 _ => None,
27272 },
27273 );
27274
27275 assert_eq!(
27276 enriched.file_path.as_deref(),
27277 Some("crates/ahri_tre_cli/src/app.rs")
27278 );
27279 assert_eq!(
27280 enriched.repository_url.as_deref(),
27281 Some("https://github.com/AHRIORG/ahri-tre-rs")
27282 );
27283 assert_eq!(enriched.commit_hash.as_deref(), Some("abcdef1"));
27284 }
27285
27286 #[test]
27287 fn transformation_source_enrichment_falls_back_when_git_is_unavailable() {
27288 let transformation = test_transformation_without_source(
27289 97,
27290 TransformationType::Transform,
27291 "fallback source provenance",
27292 );
27293 let transformation = NewTransformationRecord {
27294 file_path: Some("crates/ahri_tre_cli/src/app.rs".to_string()),
27295 ..transformation
27296 };
27297
27298 let enriched = enrich_transformation_source_with_git(&transformation, None, |_| None);
27299
27300 assert_eq!(enriched.repository_url, None);
27301 assert_eq!(enriched.commit_hash, None);
27302 assert_eq!(enriched.file_path, transformation.file_path);
27303 }
27304
27305 #[test]
27306 fn record_transformation_uses_shared_source_enrichment_path() {
27307 let repositories = TransformationRepositories::new();
27308 let service = repositories.service();
27309 let transformation = test_transformation_without_source(
27310 98,
27311 TransformationType::Ingest,
27312 "record HDSS metadata transformation",
27313 );
27314 let transformation = NewTransformationRecord {
27315 file_path: Some("crates/ahri_tre_cli/src/app.rs".to_string()),
27316 ..transformation
27317 };
27318
27319 let lineage = block_on(service.record_transformation(RecordTransformationRequest {
27320 transformation: transformation.clone(),
27321 input_version_ids: Vec::new(),
27322 output_version_ids: vec![VersionId(uuid::Uuid::new_v4())],
27323 }))
27324 .expect("shared record_transformation path should enrich provenance");
27325
27326 assert_eq!(lineage.transformation.file_path, transformation.file_path);
27327 assert_eq!(
27328 lineage.transformation.repository_url.as_deref(),
27329 Some("https://github.com/AHRIORG/ahri-tre-rs")
27330 );
27331 assert!(
27332 lineage
27333 .transformation
27334 .commit_hash
27335 .as_deref()
27336 .is_some_and(|hash| hash.len() == 7)
27337 );
27338 }
27339
27340 #[test]
27341 fn record_export_records_input_lineage_without_internal_outputs() {
27342 let repositories = TransformationRepositories::new();
27343 let service = repositories.service();
27344 let transformation = test_transformation(92, TransformationType::Export, "export dataset");
27345 let input_version_id = VersionId(uuid::Uuid::new_v4());
27346
27347 let lineage = block_on(service.record_export(RecordExportRequest {
27348 transformation: transformation.clone(),
27349 input_version_ids: vec![input_version_id],
27350 }))
27351 .expect("export provenance should record input lineage");
27352
27353 assert_eq!(lineage.transformation, transformation);
27354 assert_eq!(
27355 lineage.inputs,
27356 vec![TransformationInputLinkRecord {
27357 transformation_id: lineage.transformation.transformation_id,
27358 version_id: input_version_id,
27359 }]
27360 );
27361 assert!(lineage.outputs.is_empty());
27362 }
27363
27364 #[test]
27365 fn transform_assets_requires_transform_type_and_outputs() {
27366 let service = TransformationRepositories::new().service();
27367 let input_version_id = VersionId(uuid::Uuid::new_v4());
27368
27369 let wrong_kind = block_on(service.transform_assets(TransformAssetsRequest {
27370 transformation: test_transformation(93, TransformationType::Export, "export dataset"),
27371 input_version_ids: vec![input_version_id],
27372 output_version_ids: vec![VersionId(uuid::Uuid::new_v4())],
27373 }))
27374 .expect_err("transform_assets should reject export transformations");
27375 assert!(matches!(wrong_kind, AppError::Validation(_)));
27376
27377 let missing_output = block_on(service.transform_assets(TransformAssetsRequest {
27378 transformation: test_transformation(
27379 94,
27380 TransformationType::Transform,
27381 "missing output",
27382 ),
27383 input_version_ids: vec![input_version_id],
27384 output_version_ids: Vec::new(),
27385 }))
27386 .expect_err("transform_assets should require output versions");
27387 assert!(matches!(missing_output, AppError::Validation(_)));
27388 }
27389
27390 struct TestLakeRoot(PathBuf);
27391
27392 impl TestLakeRoot {
27393 fn path(&self) -> &Path {
27394 &self.0
27395 }
27396 }
27397
27398 impl AsRef<Path> for TestLakeRoot {
27399 fn as_ref(&self) -> &Path {
27400 self.path()
27401 }
27402 }
27403
27404 impl Deref for TestLakeRoot {
27405 type Target = Path;
27406
27407 fn deref(&self) -> &Self::Target {
27408 self.path()
27409 }
27410 }
27411
27412 impl Drop for TestLakeRoot {
27413 fn drop(&mut self) {
27414 let _ = fs::remove_dir_all(self.path());
27415 let _ = fs::remove_dir_all(self.path().with_extension("scratch"));
27416 }
27417 }
27418
27419 fn temp_lake_root_path() -> TestLakeRoot {
27420 TestLakeRoot(std::env::temp_dir().join(format!(
27421 "ahri_tre_app_lake_root_{}_{}",
27422 std::process::id(),
27423 uuid::Uuid::new_v4().simple()
27424 )))
27425 }
27426
27427 #[test]
27428 fn temp_lake_root_path_removes_root_on_drop_and_unwind() {
27429 let normal_path = {
27430 let lake_root = temp_lake_root_path();
27431 fs::create_dir_all(&lake_root).expect("test lake root should be created");
27432 lake_root.path().to_path_buf()
27433 };
27434
27435 assert!(
27436 !normal_path.exists(),
27437 "test lake root should be removed when its owner is dropped"
27438 );
27439
27440 let panic_payload = std::panic::catch_unwind(|| {
27441 let lake_root = temp_lake_root_path();
27442 fs::create_dir_all(&lake_root).expect("test lake root should be created");
27443 std::panic::panic_any(lake_root.path().to_path_buf());
27444 })
27445 .expect_err("test scope should panic");
27446 let unwound_path = *panic_payload
27447 .downcast::<PathBuf>()
27448 .expect("panic payload should contain the test lake-root path");
27449
27450 assert!(
27451 !unwound_path.exists(),
27452 "test lake root should be removed while unwinding"
27453 );
27454 }
27455
27456 fn workspace_file(relative_path: &str) -> PathBuf {
27457 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
27458 .join("..")
27459 .join("..")
27460 .join(relative_path)
27461 }
27462
27463 fn spawn_https_fixture_response(
27464 status: &str,
27465 headers: &[(&str, String)],
27466 body: Vec<u8>,
27467 ) -> (String, TestHttpsRootGuard) {
27468 spawn_https_fixture_response_with_delay(status, headers, body, None)
27469 }
27470
27471 fn spawn_https_fixture_response_with_delay(
27472 status: &str,
27473 headers: &[(&str, String)],
27474 body: Vec<u8>,
27475 delay: Option<StdDuration>,
27476 ) -> (String, TestHttpsRootGuard) {
27477 let lock = TEST_HTTPS_FIXTURE_LOCK
27478 .lock()
27479 .expect("test HTTPS fixture lock should not be poisoned");
27480 let (key, certificate) = localhost_certificate();
27481 let certificate_der = certificate
27482 .to_der()
27483 .expect("test HTTPS certificate should serialize");
27484 *TEST_HTTPS_ROOT_CERTIFICATE_DER
27485 .lock()
27486 .expect("test HTTPS certificate lock should not be poisoned") = Some(certificate_der);
27487 let guard = TestHttpsRootGuard { _lock: lock };
27488 let mut acceptor =
27489 SslAcceptor::mozilla_intermediate(SslMethod::tls()).expect("TLS acceptor should build");
27490 acceptor
27491 .set_private_key(&key)
27492 .expect("TLS private key should set");
27493 acceptor
27494 .set_certificate(&certificate)
27495 .expect("TLS certificate should set");
27496 acceptor
27497 .check_private_key()
27498 .expect("TLS private key should match certificate");
27499 let acceptor = acceptor.build();
27500 let listener = TcpListener::bind("127.0.0.1:0").expect("fixture listener should bind");
27501 let address = listener
27502 .local_addr()
27503 .expect("fixture address should resolve");
27504 let status = status.to_string();
27505 let headers = headers
27506 .iter()
27507 .map(|(name, value)| ((*name).to_string(), value.clone()))
27508 .collect::<Vec<_>>();
27509 thread::spawn(move || {
27510 let Ok((stream, _)) = listener.accept() else {
27511 return;
27512 };
27513 let Ok(mut stream) = acceptor.accept(stream) else {
27514 return;
27515 };
27516 let mut request_buffer = [0_u8; 2048];
27517 let _ = stream.read(&mut request_buffer);
27518 if let Some(delay) = delay {
27519 std::thread::sleep(delay);
27520 }
27521 let has_content_length = headers
27522 .iter()
27523 .any(|(name, _)| name.eq_ignore_ascii_case("content-length"));
27524 let mut response = format!("HTTP/1.1 {status}\r\nConnection: close\r\n").into_bytes();
27525 if !has_content_length {
27526 response
27527 .extend_from_slice(format!("Content-Length: {}\r\n", body.len()).as_bytes());
27528 }
27529 for (name, value) in headers {
27530 response.extend_from_slice(format!("{name}: {value}\r\n").as_bytes());
27531 }
27532 response.extend_from_slice(b"\r\n");
27533 response.extend_from_slice(&body);
27534 let _ = stream.write_all(&response);
27535 let _ = stream.flush();
27536 });
27537 (format!("https://localhost:{}", address.port()), guard)
27538 }
27539
27540 fn localhost_certificate() -> (PKey<Private>, X509) {
27541 let rsa = Rsa::generate(2048).expect("test RSA key should generate");
27542 let key = PKey::from_rsa(rsa).expect("test private key should build");
27543 let mut name = X509NameBuilder::new().expect("test certificate name should build");
27544 name.append_entry_by_text("CN", "localhost")
27545 .expect("test certificate CN should set");
27546 let name = name.build();
27547 let mut serial = BigNum::new().expect("test serial should allocate");
27548 serial
27549 .rand(128, MsbOption::MAYBE_ZERO, false)
27550 .expect("test serial should randomize");
27551 let serial = serial
27552 .to_asn1_integer()
27553 .expect("test serial should convert");
27554 let mut builder = X509::builder().expect("test certificate should build");
27555 builder.set_version(2).expect("test version should set");
27556 builder
27557 .set_serial_number(&serial)
27558 .expect("test serial should set");
27559 builder
27560 .set_subject_name(&name)
27561 .expect("test subject should set");
27562 builder
27563 .set_issuer_name(&name)
27564 .expect("test issuer should set");
27565 builder.set_pubkey(&key).expect("test pubkey should set");
27566 builder
27567 .set_not_before(Asn1Time::days_from_now(0).unwrap().as_ref())
27568 .expect("test not_before should set");
27569 builder
27570 .set_not_after(Asn1Time::days_from_now(1).unwrap().as_ref())
27571 .expect("test not_after should set");
27572 let basic_constraints = BasicConstraints::new()
27573 .critical()
27574 .ca()
27575 .build()
27576 .expect("test basic constraints should build");
27577 builder
27578 .append_extension(basic_constraints)
27579 .expect("test basic constraints should append");
27580 let key_usage = KeyUsage::new()
27581 .digital_signature()
27582 .key_encipherment()
27583 .key_cert_sign()
27584 .build()
27585 .expect("test key usage should build");
27586 builder
27587 .append_extension(key_usage)
27588 .expect("test key usage should append");
27589 let subject_alt_name = SubjectAlternativeName::new()
27590 .dns("localhost")
27591 .ip("127.0.0.1")
27592 .build(&builder.x509v3_context(None, None))
27593 .expect("test SAN should build");
27594 builder
27595 .append_extension(subject_alt_name)
27596 .expect("test SAN should append");
27597 builder
27598 .sign(&key, MessageDigest::sha256())
27599 .expect("test certificate should sign");
27600 (key, builder.build())
27601 }
27602
27603 fn test_lake_session(lake_root: &Path) -> DataStoreSession {
27604 fs::create_dir_all(lake_root).expect("test lake root should be created");
27605 let scratch_root = lake_root.with_extension("scratch");
27606 fs::create_dir_all(&scratch_root).unwrap();
27607 fs::set_permissions(&scratch_root, fs::Permissions::from_mode(0o700)).unwrap();
27608 let scratch = ahri_tre_lake::TrustedScratch::open(&scratch_root, [lake_root]).unwrap();
27609 let attempt = scratch
27610 .create_attempt(
27611 ahri_tre_lake::ScratchAttemptId::new(&uuid::Uuid::new_v4().simple().to_string())
27612 .unwrap(),
27613 )
27614 .unwrap();
27615 DataStoreSession::new_lake_only_for_test(
27616 test_runtime(lake_root),
27617 duckdb::Connection::open_in_memory().expect("test DuckDB connection should open"),
27618 )
27619 .with_scratch_attempt(Some(attempt))
27620 }
27621
27622 fn test_lake_session_with_scratch(lake_root: &Path, scratch_root: &Path) -> DataStoreSession {
27623 fs::create_dir_all(lake_root).expect("test lake root should be created");
27624 fs::create_dir(scratch_root).expect("test scratch root should be created");
27625 fs::set_permissions(scratch_root, fs::Permissions::from_mode(0o700))
27626 .expect("test scratch mode should be set");
27627 let scratch = ahri_tre_lake::TrustedScratch::open(scratch_root, [lake_root])
27628 .expect("test scratch should open");
27629 let attempt = scratch
27630 .create_attempt(
27631 ahri_tre_lake::ScratchAttemptId::new("0123456789abcdef0123456789abcdef")
27632 .expect("test attempt ID should be valid"),
27633 )
27634 .expect("test scratch attempt should open");
27635 DataStoreSession::new_lake_only_for_test(
27636 test_runtime(lake_root),
27637 duckdb::Connection::open_in_memory().expect("test DuckDB connection should open"),
27638 )
27639 .with_scratch_attempt(Some(attempt))
27640 }
27641
27642 fn test_lake_session_with_encryption(
27643 lake_root: &Path,
27644 scratch_root: &Path,
27645 encryption_mode: EncryptionMode,
27646 ) -> DataStoreSession {
27647 fs::create_dir_all(lake_root).expect("test lake root should be created");
27648 fs::create_dir(scratch_root).expect("test scratch root should be created");
27649 fs::set_permissions(scratch_root, fs::Permissions::from_mode(0o700))
27650 .expect("test scratch mode should be set");
27651 let scratch = ahri_tre_lake::TrustedScratch::open(scratch_root, [lake_root])
27652 .expect("test scratch should open");
27653 let attempt = scratch
27654 .create_attempt(
27655 ahri_tre_lake::ScratchAttemptId::new("fedcba9876543210fedcba9876543210")
27656 .expect("test attempt ID should be valid"),
27657 )
27658 .expect("test scratch attempt should open");
27659 let mut runtime = test_runtime(lake_root);
27660 runtime.profile.lake.auth.encryption_mode = encryption_mode;
27661 runtime.lake.detected_encryption_mode = encryption_mode;
27662 DataStoreSession::new_lake_only_for_test(
27663 runtime,
27664 duckdb::Connection::open_in_memory().expect("test DuckDB connection should open"),
27665 )
27666 .with_scratch_attempt(Some(attempt))
27667 }
27668
27669 #[test]
27670 fn file_asset_encryption_is_independent_of_ducklake_encryption() {
27671 let repositories = AssetWorkflowRepositories::new();
27672 let service = repositories.service();
27673 let study = test_study("binding_encrypt_policy_study");
27674 repositories.seed_study(study.clone());
27675 let lake_root = temp_lake_root_path();
27676 fs::create_dir_all(&lake_root).expect("test lake root should be created");
27677 let source_path = lake_root.join("binding-policy.csv");
27678 fs::write(&source_path, b"id,label\n1,alpha\n").expect("source fixture should write");
27679
27680 let scratch_root = temp_lake_root_path();
27681 let server_managed_session = test_lake_session_with_encryption(
27682 &lake_root,
27683 scratch_root.path(),
27684 EncryptionMode::ServerManaged,
27685 );
27686 let encrypted = block_on(service.ingest_file_for_session(
27687 &server_managed_session,
27688 IngestFileRequest {
27689 classification: ahri_tre_types::IngestClassification::derived_high(),
27690 study_id: study.study_id,
27691 asset_id: None,
27692 asset_name: nc_name("binding_policy_encrypted"),
27693 version_id: None,
27694 source: IngestFileSource::LocalPath {
27695 path: source_path.display().to_string(),
27696 },
27697 lake_root: "ignored-by-session".to_string(),
27698 edam_format: Some("EDAM:format_3752".to_string()),
27699 compress: false,
27700 encrypt: None,
27701 description: None,
27702 agent_instructions: None,
27703 version_note: None,
27704 transformation: test_transformation(
27705 91_001,
27706 TransformationType::Ingest,
27707 "binding policy encrypted ingest",
27708 ),
27709 },
27710 ))
27711 .expect("session ingest should inherit encrypted binding policy");
27712 assert_eq!(encrypted.datafile.encrypted, Some(true));
27713
27714 let unencrypted = block_on(service.ingest_file_for_session(
27715 &server_managed_session,
27716 IngestFileRequest {
27717 classification: ahri_tre_types::IngestClassification::derived_high(),
27718 study_id: study.study_id,
27719 asset_id: None,
27720 asset_name: nc_name("binding_policy_unencrypted"),
27721 version_id: None,
27722 source: IngestFileSource::LocalPath {
27723 path: source_path.display().to_string(),
27724 },
27725 lake_root: "ignored-by-session".to_string(),
27726 edam_format: Some("EDAM:format_3752".to_string()),
27727 compress: false,
27728 encrypt: Some(false),
27729 description: None,
27730 agent_instructions: None,
27731 version_note: None,
27732 transformation: test_transformation(
27733 91_002,
27734 TransformationType::Ingest,
27735 "binding policy rejected ingest",
27736 ),
27737 },
27738 ))
27739 .expect("explicit File Asset encryption opt-out should remain independent");
27740 assert_eq!(unencrypted.datafile.encrypted, Some(false));
27741 assert_eq!(unencrypted.datafile.encryption_algorithm, None);
27742 assert_eq!(unencrypted.datafile.encryption_key, None);
27743
27744 let _ = fs::remove_dir_all(&lake_root);
27745 }
27746
27747 fn materialize_json_datafile_with_no_registrations(
27748 suffix: &str,
27749 source_file_name: &str,
27750 source_text: &str,
27751 input_format: &str,
27752 json_format: Option<&str>,
27753 id_base: i64,
27754 ) -> DatasetMaterialization {
27755 let repositories = AssetWorkflowRepositories::new();
27756 let service = repositories.service();
27757 let study = test_study(&format!("{suffix}_study"));
27758 repositories.seed_study(study.clone());
27759 let domain = test_domain(90_500 + id_base, &format!("{suffix}_domain"));
27760 repositories.seed_study_domain(study.study_id, domain);
27761
27762 let lake_root = temp_lake_root_path();
27763 fs::create_dir_all(&lake_root).expect("JSON fallback lake root should be created");
27764 let source_path = lake_root.join(source_file_name);
27765 fs::write(&source_path, source_text).expect("JSON fallback source should write");
27766 let ingested = block_on(service.ingest_file(IngestFileRequest {
27767 classification: ahri_tre_types::IngestClassification::derived_high(),
27768 study_id: study.study_id,
27769 asset_id: None,
27770 asset_name: nc_name(&format!("{suffix}_datafile")),
27771 version_id: None,
27772 source: IngestFileSource::LocalPath {
27773 path: source_path.display().to_string(),
27774 },
27775 lake_root: lake_root.display().to_string(),
27776 edam_format: Some(input_format.to_string()),
27777 compress: true,
27778 encrypt: None,
27779 description: Some("Managed JSON datafile".to_string()),
27780 agent_instructions: None,
27781 version_note: Some("Initial managed JSON datafile".to_string()),
27782 transformation: test_transformation(
27783 id_base,
27784 TransformationType::Ingest,
27785 "ingest JSON as datafile",
27786 ),
27787 }))
27788 .expect("JSON source datafile ingest should succeed");
27789
27790 let mut session = test_lake_session(&lake_root);
27791 let materialized = block_on(service.datafile_to_dataset(
27792 &mut session,
27793 DataFileToDatasetRequest {
27794 study_id: study.study_id,
27795 metadata_domain_id: None,
27796 dataset_asset_id: None,
27797 dataset_name: nc_name(&format!("{suffix}_dataset")),
27798 dataset_version_id: None,
27799 source_datafile_id: ingested.datafile.datafile_id,
27800 input_format: Some(input_format.to_string()),
27801 header: None,
27802 delimiter: None,
27803 null_strings: Vec::new(),
27804 json_format: json_format.map(str::to_string),
27805 sheet: None,
27806 description: Some("Dataset loaded from a managed JSON datafile".to_string()),
27807 agent_instructions: None,
27808 version_note: Some("Initial JSON datafile-derived dataset".to_string()),
27809 transformation: test_transformation(
27810 id_base + 1,
27811 TransformationType::Transform,
27812 "transform JSON datafile to dataset",
27813 ),
27814 strict_metadata: false,
27815 sample_rows: None,
27816 vocabulary_threshold: None,
27817 force_metadata_updates: false,
27818 variable_registrations: Vec::new(),
27819 },
27820 ))
27821 .expect("JSON datafile-to-dataset should succeed");
27822
27823 let _ = fs::remove_dir_all(&lake_root);
27824 materialized
27825 }
27826
27827 struct CsvProfilingOptions {
27828 null_strings: Vec<String>,
27829 sample_rows: Option<usize>,
27830 vocabulary_threshold: Option<usize>,
27831 force_metadata_updates: bool,
27832 }
27833
27834 fn materialize_csv_datafile_with_profiling<F>(
27835 suffix: &str,
27836 id_base: i64,
27837 csv: &str,
27838 options: CsvProfilingOptions,
27839 seed_existing_metadata: F,
27840 ) -> Result<DatasetMaterialization, AppError>
27841 where
27842 F: FnOnce(&AssetWorkflowRepositories, &DomainRecord),
27843 {
27844 let repositories = AssetWorkflowRepositories::new();
27845 let service = repositories.service();
27846 let study = test_study(&format!("{suffix}_study"));
27847 repositories.seed_study(study.clone());
27848 let domain = test_domain(92_000 + id_base, &format!("{suffix}_domain"));
27849 repositories.seed_study_domain(study.study_id, domain.clone());
27850 seed_existing_metadata(&repositories, &domain);
27851
27852 let lake_root = temp_lake_root_path();
27853 fs::create_dir_all(&lake_root).expect("CSV profiling lake root should be created");
27854 let source_path = lake_root.join("profiled_source.csv");
27855 fs::write(&source_path, csv).expect("CSV profiling source should write");
27856 let ingested = block_on(service.ingest_file(IngestFileRequest {
27857 classification: ahri_tre_types::IngestClassification::derived_high(),
27858 study_id: study.study_id,
27859 asset_id: None,
27860 asset_name: nc_name(&format!("{suffix}_datafile")),
27861 version_id: None,
27862 source: IngestFileSource::LocalPath {
27863 path: source_path.display().to_string(),
27864 },
27865 lake_root: lake_root.display().to_string(),
27866 edam_format: Some("csv".to_string()),
27867 compress: true,
27868 encrypt: None,
27869 description: Some("Managed CSV datafile".to_string()),
27870 agent_instructions: None,
27871 version_note: Some("Initial managed CSV datafile".to_string()),
27872 transformation: test_transformation(
27873 id_base,
27874 TransformationType::Ingest,
27875 "ingest CSV for metadata profiling",
27876 ),
27877 }))
27878 .expect("CSV source datafile ingest should succeed");
27879 let mut session = test_lake_session(&lake_root);
27880 let materialized = block_on(service.datafile_to_dataset(
27881 &mut session,
27882 DataFileToDatasetRequest {
27883 study_id: study.study_id,
27884 metadata_domain_id: None,
27885 dataset_asset_id: None,
27886 dataset_name: nc_name(&format!("{suffix}_dataset")),
27887 dataset_version_id: None,
27888 source_datafile_id: ingested.datafile.datafile_id,
27889 input_format: Some("csv".to_string()),
27890 header: Some(true),
27891 delimiter: Some(','),
27892 null_strings: options.null_strings,
27893 json_format: None,
27894 sheet: None,
27895 description: Some("Dataset loaded from managed CSV".to_string()),
27896 agent_instructions: None,
27897 version_note: Some("Initial CSV datafile-derived dataset".to_string()),
27898 transformation: test_transformation(
27899 id_base + 1,
27900 TransformationType::Transform,
27901 "transform CSV datafile to dataset",
27902 ),
27903 strict_metadata: false,
27904 sample_rows: options.sample_rows,
27905 vocabulary_threshold: options.vocabulary_threshold,
27906 force_metadata_updates: options.force_metadata_updates,
27907 variable_registrations: Vec::new(),
27908 },
27909 ));
27910
27911 let _ = fs::remove_dir_all(&lake_root);
27912 materialized
27913 }
27914
27915 fn materialize_parquet_metadata_datafile<F>(
27916 suffix: &str,
27917 id_base: i64,
27918 strict_metadata: bool,
27919 seed_existing_metadata: F,
27920 ) -> Result<DatasetMaterialization, AppError>
27921 where
27922 F: FnOnce(&AssetWorkflowRepositories, &DomainRecord),
27923 {
27924 let repositories = AssetWorkflowRepositories::new();
27925 let service = repositories.service();
27926 let study = test_study(&format!("{suffix}_study"));
27927 repositories.seed_study(study.clone());
27928 let domain = test_domain(90_700 + id_base, &format!("{suffix}_domain"));
27929 repositories.seed_study_domain(study.study_id, domain.clone());
27930 seed_existing_metadata(&repositories, &domain);
27931
27932 let lake_root = temp_lake_root_path();
27933 fs::create_dir_all(&lake_root).expect("Parquet conflict lake root should be created");
27934 let source_path = lake_root.join("metadata_source.parquet");
27935 write_parquet_metadata_fixture(&source_path);
27936 let ingested = block_on(service.ingest_file(IngestFileRequest {
27937 classification: ahri_tre_types::IngestClassification::derived_high(),
27938 study_id: study.study_id,
27939 asset_id: None,
27940 asset_name: nc_name(&format!("{suffix}_datafile")),
27941 version_id: None,
27942 source: IngestFileSource::LocalPath {
27943 path: source_path.display().to_string(),
27944 },
27945 lake_root: lake_root.display().to_string(),
27946 edam_format: Some("parquet".to_string()),
27947 compress: true,
27948 encrypt: None,
27949 description: Some("Managed Parquet datafile".to_string()),
27950 agent_instructions: None,
27951 version_note: Some("Initial managed Parquet datafile".to_string()),
27952 transformation: test_transformation(
27953 id_base,
27954 TransformationType::Ingest,
27955 "ingest Parquet for metadata conflict",
27956 ),
27957 }))
27958 .expect("Parquet source datafile ingest should succeed");
27959 let mut session = test_lake_session(&lake_root);
27960 let materialized = block_on(service.datafile_to_dataset(
27961 &mut session,
27962 DataFileToDatasetRequest {
27963 study_id: study.study_id,
27964 metadata_domain_id: None,
27965 dataset_asset_id: None,
27966 dataset_name: nc_name(&format!("{suffix}_dataset")),
27967 dataset_version_id: None,
27968 source_datafile_id: ingested.datafile.datafile_id,
27969 input_format: Some("parquet".to_string()),
27970 header: None,
27971 delimiter: None,
27972 null_strings: Vec::new(),
27973 json_format: None,
27974 sheet: None,
27975 description: Some("Dataset loaded from managed Parquet".to_string()),
27976 agent_instructions: None,
27977 version_note: Some("Initial Parquet datafile-derived dataset".to_string()),
27978 transformation: test_transformation(
27979 id_base + 1,
27980 TransformationType::Transform,
27981 "transform Parquet datafile to dataset",
27982 ),
27983 strict_metadata,
27984 sample_rows: None,
27985 vocabulary_threshold: None,
27986 force_metadata_updates: false,
27987 variable_registrations: Vec::new(),
27988 },
27989 ));
27990
27991 let _ = fs::remove_dir_all(&lake_root);
27992 materialized
27993 }
27994
27995 fn materialize_arrow_ipc_metadata_datafile<F>(
27996 suffix: &str,
27997 id_base: i64,
27998 strict_metadata: bool,
27999 seed_existing_metadata: F,
28000 ) -> Result<DatasetMaterialization, AppError>
28001 where
28002 F: FnOnce(&AssetWorkflowRepositories, &DomainRecord),
28003 {
28004 let repositories = AssetWorkflowRepositories::new();
28005 let service = repositories.service();
28006 let study = test_study(&format!("{suffix}_study"));
28007 repositories.seed_study(study.clone());
28008 let domain = test_domain(91_000 + id_base, &format!("{suffix}_domain"));
28009 repositories.seed_study_domain(study.study_id, domain.clone());
28010 seed_existing_metadata(&repositories, &domain);
28011
28012 let lake_root = temp_lake_root_path();
28013 fs::create_dir_all(&lake_root).expect("Arrow IPC conflict lake root should be created");
28014 let source_path = lake_root.join("metadata_source.arrow");
28015 write_arrow_ipc_metadata_fixture(&source_path);
28016 let ingested = block_on(service.ingest_file(IngestFileRequest {
28017 classification: ahri_tre_types::IngestClassification::derived_high(),
28018 study_id: study.study_id,
28019 asset_id: None,
28020 asset_name: nc_name(&format!("{suffix}_datafile")),
28021 version_id: None,
28022 source: IngestFileSource::LocalPath {
28023 path: source_path.display().to_string(),
28024 },
28025 lake_root: lake_root.display().to_string(),
28026 edam_format: Some("application/vnd.apache.arrow.file".to_string()),
28027 compress: true,
28028 encrypt: None,
28029 description: Some("Managed Arrow IPC datafile".to_string()),
28030 agent_instructions: None,
28031 version_note: Some("Initial managed Arrow IPC datafile".to_string()),
28032 transformation: test_transformation(
28033 id_base,
28034 TransformationType::Ingest,
28035 "ingest Arrow IPC for metadata conflict",
28036 ),
28037 }))
28038 .expect("Arrow IPC source datafile ingest should succeed");
28039 let mut session = test_lake_session(&lake_root);
28040 let materialized = block_on(service.datafile_to_dataset(
28041 &mut session,
28042 DataFileToDatasetRequest {
28043 study_id: study.study_id,
28044 metadata_domain_id: None,
28045 dataset_asset_id: None,
28046 dataset_name: nc_name(&format!("{suffix}_dataset")),
28047 dataset_version_id: None,
28048 source_datafile_id: ingested.datafile.datafile_id,
28049 input_format: Some("arrow".to_string()),
28050 header: None,
28051 delimiter: None,
28052 null_strings: Vec::new(),
28053 json_format: None,
28054 sheet: None,
28055 description: Some("Dataset loaded from managed Arrow IPC".to_string()),
28056 agent_instructions: None,
28057 version_note: Some("Initial Arrow IPC datafile-derived dataset".to_string()),
28058 transformation: test_transformation(
28059 id_base + 1,
28060 TransformationType::Transform,
28061 "transform Arrow IPC datafile to dataset",
28062 ),
28063 strict_metadata,
28064 sample_rows: None,
28065 vocabulary_threshold: None,
28066 force_metadata_updates: false,
28067 variable_registrations: Vec::new(),
28068 },
28069 ));
28070
28071 let _ = fs::remove_dir_all(&lake_root);
28072 materialized
28073 }
28074
28075 fn seed_status_code_variable_and_vocabulary(
28076 repositories: &AssetWorkflowRepositories,
28077 domain: &DomainRecord,
28078 variable_description: &str,
28079 vocabulary_description: Option<&str>,
28080 vocabulary_items: Vec<VocabularyItemRecord>,
28081 ) {
28082 let vocabulary_id = VocabularyId(vocabulary_items[0].vocabulary_id.0);
28083 repositories.seed_vocabulary(VocabularyRecord {
28084 vocabulary_id,
28085 domain_id: domain.domain_id,
28086 name: nc_name("status_vocab"),
28087 description: vocabulary_description.map(str::to_string),
28088 });
28089 repositories.seed_vocabulary_items(vocabulary_id, vocabulary_items);
28090 let mut variable = test_variable(
28091 10_600 + vocabulary_id.0,
28092 domain.domain_id.0,
28093 "status_code",
28094 KeyRole::None,
28095 Some(vocabulary_id),
28096 );
28097 variable.value_type_id = ValueTypeId(7);
28098 variable.description = Some(variable_description.to_string());
28099 repositories.seed_variable(variable);
28100 }
28101
28102 fn matching_status_vocabulary_items(vocabulary_id: i64) -> Vec<VocabularyItemRecord> {
28103 vec![
28104 VocabularyItemRecord {
28105 vocabulary_item_id: VocabularyItemId(vocabulary_id),
28106 vocabulary_id: VocabularyId(vocabulary_id),
28107 value: 1,
28108 code: "A".to_string(),
28109 description: Some("Alive".to_string()),
28110 },
28111 VocabularyItemRecord {
28112 vocabulary_item_id: VocabularyItemId(vocabulary_id + 1),
28113 vocabulary_id: VocabularyId(vocabulary_id),
28114 value: 2,
28115 code: "D".to_string(),
28116 description: Some("Dead".to_string()),
28117 },
28118 ]
28119 }
28120
28121 fn assert_imported_shape_variable(
28122 materialized: &DatasetMaterialization,
28123 name: &str,
28124 value_type_id: ValueTypeId,
28125 message: &str,
28126 ) {
28127 let variable = materialized
28128 .variables
28129 .iter()
28130 .find(|registered| registered.variable.name.as_str() == name)
28131 .unwrap_or_else(|| panic!("{message}: missing variable {name}"));
28132 assert_eq!(variable.variable.value_type_id, value_type_id, "{message}");
28133 assert_eq!(variable.variable.description, None, "{message}");
28134 assert_eq!(variable.variable.note, None, "{message}");
28135 assert!(variable.vocabulary.is_none(), "{message}");
28136 assert!(variable.vocabulary_items.is_empty(), "{message}");
28137 assert_eq!(variable.link.dataset_id, materialized.dataset.dataset_id);
28138 }
28139
28140 fn write_parquet_metadata_fixture(path: &Path) {
28141 let table = metadata_fixture_table();
28142 ahri_tre_tabular::write_parquet_file(path, &table)
28143 .expect("Parquet metadata fixture should write");
28144 }
28145
28146 fn write_arrow_ipc_metadata_fixture(path: &Path) {
28147 let table = metadata_fixture_table();
28148 let bytes = ahri_tre_tabular::write_ipc_file(&table)
28149 .expect("Arrow IPC metadata fixture should write");
28150 fs::write(path, bytes).expect("Arrow IPC metadata fixture should persist");
28151 }
28152
28153 fn metadata_fixture_table() -> ahri_tre_tabular::ArrowTable {
28154 let batch = ahri_tre_tabular::columns_to_record_batch(vec![
28155 ahri_tre_tabular::TabularColumn::Utf8 {
28156 name: "status".to_string(),
28157 values: vec![Some("A".to_string()), Some("D".to_string())],
28158 },
28159 ahri_tre_tabular::TabularColumn::Utf8 {
28160 name: "sex code".to_string(),
28161 values: vec![Some("F".to_string()), Some("M".to_string())],
28162 },
28163 ahri_tre_tabular::TabularColumn::Int64 {
28164 name: "age".to_string(),
28165 values: vec![Some(34), Some(41)],
28166 },
28167 ])
28168 .expect("Parquet metadata batch should build");
28169 let fields = vec![
28170 batch
28171 .schema()
28172 .field_with_name("status")
28173 .expect("status field should exist")
28174 .clone()
28175 .with_metadata(
28176 [
28177 (
28178 metadata_keys::VARIABLE_NAME.to_string(),
28179 "status_code".to_string(),
28180 ),
28181 (
28182 metadata_keys::DESCRIPTION.to_string(),
28183 "AHRI TRE status description".to_string(),
28184 ),
28185 (
28186 "description".to_string(),
28187 "Conventional status description".to_string(),
28188 ),
28189 (
28190 metadata_keys::VOCABULARY_NAME.to_string(),
28191 "status_vocab".to_string(),
28192 ),
28193 (
28194 metadata_keys::VOCABULARY_DESCRIPTION.to_string(),
28195 "AHRI TRE status vocabulary".to_string(),
28196 ),
28197 (
28198 metadata_keys::VOCABULARY_ITEMS.to_string(),
28199 r#"[{"code":"A","label":"Alive"},{"code":"D","description":"Dead"}]"#
28200 .to_string(),
28201 ),
28202 ]
28203 .into_iter()
28204 .collect(),
28205 ),
28206 batch
28207 .schema()
28208 .field_with_name("sex code")
28209 .expect("sex field should exist")
28210 .clone()
28211 .with_metadata(
28212 [
28213 ("label".to_string(), "Sex at baseline".to_string()),
28214 (
28215 "value_labels".to_string(),
28216 r#"{"F":"Female","M":"Male"}"#.to_string(),
28217 ),
28218 ("owner".to_string(), "ignored".to_string()),
28219 ]
28220 .into_iter()
28221 .collect(),
28222 ),
28223 batch
28224 .schema()
28225 .field_with_name("age")
28226 .expect("age field should exist")
28227 .clone(),
28228 ];
28229 ahri_tre_tabular::ArrowTable::from_batches(vec![batch])
28230 .and_then(|table| table.with_schema(ahri_tre_tabular::Schema::new(fields)))
28231 .expect("metadata fixture table should build")
28232 }
28233
28234 fn xlsx_reader_unavailable(error: &AppError) -> bool {
28235 let message = error.to_string().to_ascii_lowercase();
28236 message.contains("read_xlsx")
28237 && (message.contains("does not exist")
28238 || message.contains("extension")
28239 || message.contains("install"))
28240 }
28241
28242 fn write_minimal_xlsx_fixture(path: &Path) -> Result<(), String> {
28243 let root = path.with_extension("xlsx_parts");
28244 let _ = fs::remove_file(path);
28245 let _ = fs::remove_dir_all(&root);
28246 fs::create_dir_all(root.join("_rels")).map_err(|error| error.to_string())?;
28247 fs::create_dir_all(root.join("xl").join("_rels")).map_err(|error| error.to_string())?;
28248 fs::create_dir_all(root.join("xl").join("worksheets"))
28249 .map_err(|error| error.to_string())?;
28250 fs::write(
28251 root.join("[Content_Types].xml"),
28252 r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
28253<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
28254 <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
28255 <Default Extension="xml" ContentType="application/xml"/>
28256 <Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
28257 <Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
28258</Types>"#,
28259 )
28260 .map_err(|error| error.to_string())?;
28261 fs::write(
28262 root.join("_rels").join(".rels"),
28263 r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
28264<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
28265 <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
28266</Relationships>"#,
28267 )
28268 .map_err(|error| error.to_string())?;
28269 fs::write(
28270 root.join("xl").join("workbook.xml"),
28271 r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
28272<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
28273 <sheets>
28274 <sheet name="Sheet1" sheetId="1" r:id="rId1"/>
28275 </sheets>
28276</workbook>"#,
28277 )
28278 .map_err(|error| error.to_string())?;
28279 fs::write(
28280 root.join("xl").join("_rels").join("workbook.xml.rels"),
28281 r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
28282<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
28283 <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
28284</Relationships>"#,
28285 )
28286 .map_err(|error| error.to_string())?;
28287 fs::write(
28288 root.join("xl").join("worksheets").join("sheet1.xml"),
28289 r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
28290<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
28291 <sheetData>
28292 <row r="1">
28293 <c r="A1" t="inlineStr"><is><t>country</t></is></c>
28294 <c r="B1" t="inlineStr"><is><t>year</t></is></c>
28295 <c r="C1" t="inlineStr"><is><t>population</t></is></c>
28296 </row>
28297 <row r="2">
28298 <c r="A2" t="inlineStr"><is><t>ZA</t></is></c>
28299 <c r="B2"><v>2020</v></c>
28300 <c r="C2"><v>100</v></c>
28301 </row>
28302 <row r="3">
28303 <c r="A3" t="inlineStr"><is><t>MZ</t></is></c>
28304 <c r="B3"><v>2021</v></c>
28305 <c r="C3"><v>125</v></c>
28306 </row>
28307 </sheetData>
28308</worksheet>"#,
28309 )
28310 .map_err(|error| error.to_string())?;
28311 let status = Command::new("zip")
28312 .arg("-q")
28313 .arg("-r")
28314 .arg(path)
28315 .arg(".")
28316 .current_dir(&root)
28317 .status()
28318 .map_err(|error| error.to_string())?;
28319 let _ = fs::remove_dir_all(&root);
28320 if status.success() {
28321 Ok(())
28322 } else {
28323 Err(format!("zip exited with status {status}"))
28324 }
28325 }
28326
28327 fn seed_meta_test_duckdb(path: &Path) {
28328 if let Some(parent) = path.parent() {
28329 fs::create_dir_all(parent).expect("source DuckDB parent should be created");
28330 }
28331 if path.exists() {
28332 let _ = fs::remove_file(path);
28333 }
28334 let connection = duckdb::Connection::open(path).expect("source DuckDB should open");
28335 connection
28336 .execute_batch(
28337 "
28338 CREATE TABLE causes (
28339 code INTEGER PRIMARY KEY,
28340 label VARCHAR NOT NULL,
28341 description VARCHAR
28342 );
28343 CREATE TABLE deaths (
28344 death_id INTEGER PRIMARY KEY,
28345 site_id INTEGER NOT NULL,
28346 cause INTEGER
28347 );
28348 CREATE TABLE sites (
28349 site_id INTEGER PRIMARY KEY,
28350 source_id INTEGER
28351 );
28352 CREATE TABLE sources (
28353 id INTEGER PRIMARY KEY,
28354 name VARCHAR NOT NULL
28355 );
28356 INSERT INTO causes VALUES
28357 (1, 'Natural', 'Died from natural causes'),
28358 (2, 'Unnatural', 'Did not die from natural causes');
28359 INSERT INTO deaths VALUES
28360 (1, 10, 1),
28361 (2, 10, 1),
28362 (3, 20, 2);
28363 INSERT INTO sites VALUES (10, 1), (20, 2);
28364 INSERT INTO sources VALUES (1, 'Alpha'), (2, 'Beta');
28365 ",
28366 )
28367 .expect("source DuckDB fixture should initialize");
28368 }
28369
28370 fn seed_meta_test_sqlite(path: &Path) -> Result<(), duckdb::Error> {
28371 if let Some(parent) = path.parent() {
28372 fs::create_dir_all(parent).expect("source SQLite parent should be created");
28373 }
28374 if path.exists() {
28375 let _ = fs::remove_file(path);
28376 }
28377 let connection = duckdb::Connection::open_in_memory()?;
28378 connection.execute_batch(&format!(
28379 "INSTALL sqlite; LOAD sqlite; ATTACH {} AS sqlite_db (TYPE sqlite);",
28380 sql_test_literal(&path.display().to_string())
28381 ))?;
28382 connection.execute_batch(
28383 "
28384 CREATE TABLE sqlite_db.causes (
28385 code INTEGER PRIMARY KEY,
28386 label TEXT NOT NULL,
28387 description TEXT
28388 );
28389 CREATE TABLE sqlite_db.deaths (
28390 death_id INTEGER PRIMARY KEY,
28391 site_id INTEGER NOT NULL,
28392 cause INTEGER
28393 );
28394 INSERT INTO sqlite_db.causes VALUES
28395 (1, 'Natural', 'Died from natural causes'),
28396 (2, 'Unnatural', 'Did not die from natural causes');
28397 INSERT INTO sqlite_db.deaths VALUES
28398 (1, 10, 1),
28399 (2, 10, 1),
28400 (3, 20, 2);
28401 ",
28402 )?;
28403 Ok(())
28404 }
28405
28406 fn sql_test_literal(value: &str) -> String {
28407 format!("'{}'", value.replace('\'', "''"))
28408 }
28409
28410 fn test_runtime(lake_root: &Path) -> DataStoreRuntime {
28411 let lake_root = lake_root.display().to_string();
28412 let profile = DataStoreProfile {
28413 server: "tre-postgres".to_string(),
28414 port: 5432,
28415 dbname: "tre".to_string(),
28416 sslmode: None,
28417 lake: LakeAttachProfile {
28418 lake_data: lake_root.clone(),
28419 lake_db: "lake_catalog".to_string(),
28420 catalog_schema: None,
28421 auth: LakeAuthProfile {
28422 lake_user: "lake_user".to_string(),
28423 encryption_mode: EncryptionMode::None,
28424 },
28425 },
28426 };
28427
28428 DataStoreRuntime::new(
28429 profile,
28430 PgStoreConnection {
28431 connection_description: "test metadata connection".to_string(),
28432 },
28433 DuckLakeConnection {
28434 alias: "test_lake".to_string(),
28435 data_path: lake_root.clone(),
28436 staging_path: Path::new(&lake_root)
28437 .join("__tre_duckdb_stage")
28438 .display()
28439 .to_string(),
28440 tmp_path: Path::new(&lake_root)
28441 .join("__tre_duckdb_stage")
28442 .display()
28443 .to_string(),
28444 test_runs_path: Path::new(&lake_root)
28445 .join("test-runs")
28446 .display()
28447 .to_string(),
28448 catalog_database: "lake_catalog".to_string(),
28449 catalog_schema: None,
28450 attach_description: "test in-memory DuckDB connection".to_string(),
28451 detected_encryption_mode: EncryptionMode::None,
28452 automatic_migration: false,
28453 snapshot_count: 0,
28454 table_count: 0,
28455 catalog_type: "in_memory".to_string(),
28456 extension_version: "test".to_string(),
28457 },
28458 )
28459 }
28460
28461 fn dataset_row_count(session: &DataStoreSession, schema_name: &str, table_name: &str) -> i64 {
28462 session
28463 .lake_connection()
28464 .query_row(
28465 &format!(
28466 "SELECT count(*) FROM {}",
28467 quote_qualified_relation(schema_name, table_name)
28468 ),
28469 [],
28470 |row| row.get(0),
28471 )
28472 .expect("dataset table should be queryable")
28473 }
28474
28475 fn dataset_table_exists(
28476 session: &DataStoreSession,
28477 schema_name: &str,
28478 table_name: &str,
28479 ) -> bool {
28480 session
28481 .lake_connection()
28482 .prepare(&format!(
28483 "SELECT 1 FROM {} LIMIT 1",
28484 quote_qualified_relation(schema_name, table_name)
28485 ))
28486 .is_ok()
28487 }
28488
28489 fn quote_qualified_relation(schema_name: &str, table_name: &str) -> String {
28490 format!(
28491 "{}.{}",
28492 quote_identifier(schema_name),
28493 quote_identifier(table_name)
28494 )
28495 }
28496
28497 fn quote_identifier(value: &str) -> String {
28498 format!("\"{}\"", value.replace('"', "\"\""))
28499 }
28500
28501 struct RegistrationRepositories {
28502 state: Arc<RegistrationState>,
28503 registrations: Arc<StatefulCatalogueRegistrationRepository>,
28504 domains: Arc<StatefulDomainRepository>,
28505 studies: Arc<StatefulStudyRepository>,
28506 study_domains: Arc<StatefulStudyDomainRepository>,
28507 study_governance: Arc<StatefulStudyGovernanceRepository>,
28508 tags: Arc<StatefulRegistrationTagRepository>,
28509 }
28510
28511 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
28512 enum RegistrationFailurePoint {
28513 DomainTags,
28514 StudyTags,
28515 StudyDomain,
28516 StudyAccess,
28517 StudyCustodian,
28518 }
28519
28520 struct StatefulCatalogueRegistrationRepository {
28521 state: Arc<RegistrationState>,
28522 }
28523
28524 struct StatefulRegistrationTagRepository {
28525 state: Arc<RegistrationState>,
28526 }
28527
28528 impl StatefulCatalogueRegistrationRepository {
28529 fn save_domain(
28530 &self,
28531 domain: DomainRecord,
28532 normalized_tags: &[String],
28533 ) -> Result<DomainRecord, CoreError> {
28534 let mut domains_by_id = self
28535 .state
28536 .domains_by_id
28537 .lock()
28538 .expect("domain lock should not be poisoned");
28539 let mut domains_by_name = self
28540 .state
28541 .domains_by_name
28542 .lock()
28543 .expect("domain lock should not be poisoned");
28544 let mut tags = self
28545 .state
28546 .tags
28547 .lock()
28548 .expect("tag lock should not be poisoned");
28549 let mut tag_links = self
28550 .state
28551 .tag_links
28552 .lock()
28553 .expect("tag link lock should not be poisoned");
28554
28555 let mut pending_domains_by_id = domains_by_id.clone();
28556 let mut pending_domains_by_name = domains_by_name.clone();
28557 let mut pending_tags = tags.clone();
28558 let mut pending_tag_links = tag_links.clone();
28559 pending_domains_by_id.insert(domain.domain_id.0, domain.clone());
28560 pending_domains_by_name.insert(domain.name.as_str().to_string(), domain.clone());
28561 prepare_registration_tags(
28562 &mut pending_tags,
28563 &mut pending_tag_links,
28564 TagAttachmentTarget::Domain(domain.domain_id),
28565 normalized_tags,
28566 );
28567 self.state
28568 .fail_registration_at(RegistrationFailurePoint::DomainTags)?;
28569
28570 *domains_by_id = pending_domains_by_id;
28571 *domains_by_name = pending_domains_by_name;
28572 *tags = pending_tags;
28573 *tag_links = pending_tag_links;
28574 Ok(domain)
28575 }
28576
28577 fn save_study(
28578 &self,
28579 study: StudyRecord,
28580 domain_ids: &[DomainId],
28581 normalized_tags: &[String],
28582 ) -> Result<StudyRecord, CoreError> {
28583 let mut studies_by_id = self
28584 .state
28585 .studies_by_id
28586 .lock()
28587 .expect("study lock should not be poisoned");
28588 let mut studies_by_name = self
28589 .state
28590 .studies_by_name
28591 .lock()
28592 .expect("study lock should not be poisoned");
28593 let mut tags = self
28594 .state
28595 .tags
28596 .lock()
28597 .expect("tag lock should not be poisoned");
28598 let mut tag_links = self
28599 .state
28600 .tag_links
28601 .lock()
28602 .expect("tag link lock should not be poisoned");
28603 let mut study_domains = self
28604 .state
28605 .study_domains
28606 .lock()
28607 .expect("study-domain lock should not be poisoned");
28608 let mut access_grants = self
28609 .state
28610 .access_grants
28611 .lock()
28612 .expect("study access lock should not be poisoned");
28613 let mut custodians = self
28614 .state
28615 .custodians
28616 .lock()
28617 .expect("study custodian lock should not be poisoned");
28618
28619 let mut pending_studies_by_id = studies_by_id.clone();
28620 let mut pending_studies_by_name = studies_by_name.clone();
28621 let mut pending_tags = tags.clone();
28622 let mut pending_tag_links = tag_links.clone();
28623 let mut pending_study_domains = study_domains.clone();
28624 let mut pending_access_grants = access_grants.clone();
28625 let mut pending_custodians = custodians.clone();
28626 pending_studies_by_id.insert(study.study_id.0, study.clone());
28627 pending_studies_by_name.insert(study.name.as_str().to_string(), study.clone());
28628
28629 prepare_registration_tags(
28630 &mut pending_tags,
28631 &mut pending_tag_links,
28632 TagAttachmentTarget::Study(study.study_id),
28633 normalized_tags,
28634 );
28635 self.state
28636 .fail_registration_at(RegistrationFailurePoint::StudyTags)?;
28637
28638 pending_study_domains.extend(domain_ids.iter().map(|domain_id| {
28639 StudyDomainLinkRecord {
28640 study_id: study.study_id,
28641 domain_id: *domain_id,
28642 }
28643 }));
28644 self.state
28645 .fail_registration_at(RegistrationFailurePoint::StudyDomain)?;
28646
28647 if let Some(creator) = study.created_by.as_deref() {
28648 pending_access_grants.push(StudyAccessGrantRecord {
28649 study_id: study.study_id,
28650 user_id: creator.to_string(),
28651 date_granted: None,
28652 granted_by: Some(creator.to_string()),
28653 });
28654 }
28655 self.state
28656 .fail_registration_at(RegistrationFailurePoint::StudyAccess)?;
28657
28658 if let Some(creator) = study.created_by.as_deref() {
28659 pending_custodians.push(StudyCustodianLinkRecord {
28660 study_id: study.study_id,
28661 user_id: creator.to_string(),
28662 is_primary: true,
28663 date_granted: None,
28664 granted_by: Some(creator.to_string()),
28665 });
28666 }
28667 self.state
28668 .fail_registration_at(RegistrationFailurePoint::StudyCustodian)?;
28669
28670 *studies_by_id = pending_studies_by_id;
28671 *studies_by_name = pending_studies_by_name;
28672 *tags = pending_tags;
28673 *tag_links = pending_tag_links;
28674 *study_domains = pending_study_domains;
28675 *access_grants = pending_access_grants;
28676 *custodians = pending_custodians;
28677 Ok(study)
28678 }
28679 }
28680
28681 #[async_trait]
28682 impl CatalogueRegistrationRepository for StatefulCatalogueRegistrationRepository {
28683 async fn create_domain_registration(
28684 &self,
28685 domain: &NewDomainRecord,
28686 normalized_tags: &[String],
28687 ) -> Result<DomainRecord, CoreError> {
28688 let domain_id = DomainId(
28689 self.state
28690 .domains_by_id
28691 .lock()
28692 .expect("domain lock should not be poisoned")
28693 .keys()
28694 .max()
28695 .copied()
28696 .unwrap_or(0)
28697 + 1,
28698 );
28699 let domain = DomainRecord {
28700 domain_id,
28701 name: domain.name.clone(),
28702 description: domain.description.clone(),
28703 uri: domain.uri.clone(),
28704 };
28705 self.save_domain(domain, normalized_tags)
28706 }
28707
28708 async fn create_study_registration(
28709 &self,
28710 study: &NewStudyRecord,
28711 domain_ids: &[DomainId],
28712 normalized_tags: &[String],
28713 ) -> Result<StudyRecord, CoreError> {
28714 let creator = study
28715 .created_by
28716 .clone()
28717 .unwrap_or_else(|| "test_creator".to_string());
28718 let study = StudyRecord {
28719 study_id: StudyId(uuid::Uuid::new_v4()),
28720 name: study.name.clone(),
28721 description: study.description.clone(),
28722 documentation: study.documentation.clone(),
28723 agent_instructions: study.agent_instructions.clone(),
28724 external_id: study.external_id.clone(),
28725 study_type_id: study.study_type_id,
28726 date_created: study.date_created,
28727 created_by: Some(creator.clone()),
28728 };
28729 self.save_study(study, domain_ids, normalized_tags)
28730 }
28731
28732 async fn save_domain_registration(
28733 &self,
28734 domain: &DomainRecord,
28735 normalized_tags: &[String],
28736 ) -> Result<DomainRecord, CoreError> {
28737 self.save_domain(domain.clone(), normalized_tags)
28738 }
28739
28740 async fn save_study_registration(
28741 &self,
28742 study: &StudyRecord,
28743 domain_ids: &[DomainId],
28744 normalized_tags: &[String],
28745 ) -> Result<StudyRecord, CoreError> {
28746 self.save_study(study.clone(), domain_ids, normalized_tags)
28747 }
28748 }
28749
28750 #[async_trait]
28751 impl TagRepository for StatefulRegistrationTagRepository {
28752 async fn list_tags(&self) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
28753 Ok(self
28754 .state
28755 .tags
28756 .lock()
28757 .expect("tag lock should not be poisoned")
28758 .clone())
28759 }
28760
28761 async fn upsert_tag(
28762 &self,
28763 normalized_name: &str,
28764 ) -> Result<ahri_tre_types::TagRecord, CoreError> {
28765 Ok(self.state.registration_tag(normalized_name))
28766 }
28767
28768 async fn attach_tag(
28769 &self,
28770 tag_id: TagId,
28771 target: TagAttachmentTarget,
28772 ) -> Result<(), CoreError> {
28773 let mut links = self
28774 .state
28775 .tag_links
28776 .lock()
28777 .expect("tag link lock should not be poisoned");
28778 if !links.contains(&(target.clone(), tag_id)) {
28779 links.push((target, tag_id));
28780 }
28781 Ok(())
28782 }
28783
28784 async fn detach_tag(
28785 &self,
28786 tag_id: TagId,
28787 target: TagAttachmentTarget,
28788 ) -> Result<(), CoreError> {
28789 self.state
28790 .tag_links
28791 .lock()
28792 .expect("tag link lock should not be poisoned")
28793 .retain(|link| link != &(target.clone(), tag_id));
28794 Ok(())
28795 }
28796
28797 async fn list_tags_for_target(
28798 &self,
28799 target: TagAttachmentTarget,
28800 ) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
28801 let tag_ids = self
28802 .state
28803 .tag_links
28804 .lock()
28805 .expect("tag link lock should not be poisoned")
28806 .iter()
28807 .filter(|(linked_target, _)| linked_target == &target)
28808 .map(|(_, tag_id)| *tag_id)
28809 .collect::<Vec<_>>();
28810 Ok(self
28811 .state
28812 .tags
28813 .lock()
28814 .expect("tag lock should not be poisoned")
28815 .iter()
28816 .filter(|tag| tag_ids.contains(&tag.tag_id))
28817 .cloned()
28818 .collect())
28819 }
28820 }
28821
28822 impl RegistrationRepositories {
28823 fn new() -> Self {
28824 Self::with_seeded_governance_defaults(true)
28825 }
28826
28827 fn with_seeded_governance_defaults(seed_governance_defaults: bool) -> Self {
28828 let state = Arc::new(RegistrationState::new(seed_governance_defaults));
28829 Self {
28830 state: Arc::clone(&state),
28831 registrations: Arc::new(StatefulCatalogueRegistrationRepository {
28832 state: Arc::clone(&state),
28833 }),
28834 domains: Arc::new(StatefulDomainRepository {
28835 state: Arc::clone(&state),
28836 }),
28837 studies: Arc::new(StatefulStudyRepository {
28838 state: Arc::clone(&state),
28839 }),
28840 study_domains: Arc::new(StatefulStudyDomainRepository {
28841 state: Arc::clone(&state),
28842 }),
28843 study_governance: Arc::new(StatefulStudyGovernanceRepository {
28844 state: Arc::clone(&state),
28845 }),
28846 tags: Arc::new(StatefulRegistrationTagRepository { state }),
28847 }
28848 }
28849
28850 fn service(&self) -> AppService {
28851 AppService::new(AppRepositories {
28852 registrations: self.registrations.clone(),
28853 domains: self.domains.clone(),
28854 studies: self.studies.clone(),
28855 study_domains: self.study_domains.clone(),
28856 study_governance: self.study_governance.clone(),
28857 assets: Arc::new(NoopAssetRepository),
28858 variables: Arc::new(NoopVariableRepository),
28859 vocabularies: Arc::new(NoopVocabularyRepository),
28860 entities: Arc::new(NoopEntityRepository),
28861 transformations: Arc::new(NoopTransformationRepository),
28862 tags: self.tags.clone(),
28863 })
28864 }
28865
28866 fn state(&self) -> &RegistrationState {
28867 self.state.as_ref()
28868 }
28869
28870 fn fail_next_registration_at(&self, point: RegistrationFailurePoint) {
28871 *self
28872 .state
28873 .fail_next_registration
28874 .lock()
28875 .expect("registration failure lock should not be poisoned") = Some(point);
28876 }
28877 }
28878
28879 struct EntityGraphRepositories {
28880 state: Arc<EntityGraphState>,
28881 domains: Arc<StatefulEntityGraphDomainRepository>,
28882 studies: Arc<StatefulEntityGraphStudyRepository>,
28883 entities: Arc<StatefulEntityGraphRepository>,
28884 }
28885
28886 impl EntityGraphRepositories {
28887 fn new() -> Self {
28888 let state = Arc::new(EntityGraphState::default());
28889 Self {
28890 state: Arc::clone(&state),
28891 domains: Arc::new(StatefulEntityGraphDomainRepository {
28892 state: Arc::clone(&state),
28893 }),
28894 studies: Arc::new(StatefulEntityGraphStudyRepository {
28895 state: Arc::clone(&state),
28896 }),
28897 entities: Arc::new(StatefulEntityGraphRepository { state }),
28898 }
28899 }
28900
28901 fn service(&self) -> AppService {
28902 AppService::new(AppRepositories {
28903 registrations: Arc::new(NoopCatalogueRegistrationRepository),
28904 domains: self.domains.clone(),
28905 studies: self.studies.clone(),
28906 study_domains: Arc::new(NoopStudyDomainRepository),
28907 study_governance: Arc::new(NoopStudyGovernanceRepository),
28908 assets: Arc::new(NoopAssetRepository),
28909 variables: Arc::new(NoopVariableRepository),
28910 vocabularies: Arc::new(NoopVocabularyRepository),
28911 entities: self.entities.clone(),
28912 transformations: Arc::new(NoopTransformationRepository),
28913 tags: Arc::new(NoopTagRepository),
28914 })
28915 }
28916
28917 fn seed_domain(&self, domain: DomainRecord) {
28918 self.state
28919 .domains_by_id
28920 .lock()
28921 .expect("entity graph domain lock should not be poisoned")
28922 .insert(domain.domain_id.0, domain);
28923 }
28924
28925 fn seed_study(&self, study: StudyRecord) {
28926 self.state
28927 .studies_by_id
28928 .lock()
28929 .expect("entity graph study lock should not be poisoned")
28930 .insert(study.study_id.0, study);
28931 }
28932 }
28933
28934 struct AssetWorkflowRepositories {
28935 state: Arc<AssetWorkflowState>,
28936 domains: Arc<StatefulAssetDomainRepository>,
28937 studies: Arc<StatefulAssetStudyRepository>,
28938 assets: Arc<StatefulAssetRepository>,
28939 variables: Arc<StatefulVariableRepository>,
28940 vocabularies: Arc<StatefulVocabularyRepository>,
28941 study_domains: Arc<StatefulAssetStudyDomainRepository>,
28942 transformations: Arc<StatefulTransformationRepository>,
28943 }
28944
28945 impl AssetWorkflowRepositories {
28946 fn new() -> Self {
28947 let state = Arc::new(AssetWorkflowState::default());
28948 Self {
28949 state: Arc::clone(&state),
28950 domains: Arc::new(StatefulAssetDomainRepository {
28951 state: Arc::clone(&state),
28952 }),
28953 studies: Arc::new(StatefulAssetStudyRepository {
28954 state: Arc::clone(&state),
28955 }),
28956 assets: Arc::new(StatefulAssetRepository {
28957 state: Arc::clone(&state),
28958 }),
28959 variables: Arc::new(StatefulVariableRepository {
28960 state: Arc::clone(&state),
28961 }),
28962 vocabularies: Arc::new(StatefulVocabularyRepository {
28963 state: Arc::clone(&state),
28964 }),
28965 study_domains: Arc::new(StatefulAssetStudyDomainRepository {
28966 state: Arc::clone(&state),
28967 }),
28968 transformations: Arc::new(StatefulTransformationRepository {
28969 state: Arc::new(TransformationState::default()),
28970 }),
28971 }
28972 }
28973
28974 fn seed_study(&self, study: StudyRecord) {
28975 self.state
28976 .studies_by_id
28977 .lock()
28978 .expect("asset workflow study lock should not be poisoned")
28979 .insert(study.study_id.0, study);
28980 }
28981
28982 fn seed_study_domain(&self, study_id: StudyId, domain: DomainRecord) {
28983 self.seed_domain(domain.clone());
28984 self.state
28985 .domains_by_study
28986 .lock()
28987 .expect("asset workflow study-domain lock should not be poisoned")
28988 .push((study_id, domain));
28989 }
28990
28991 fn seed_domain(&self, domain: DomainRecord) {
28992 self.state
28993 .domains_by_id
28994 .lock()
28995 .expect("asset workflow domain lock should not be poisoned")
28996 .insert(domain.domain_id.0, domain);
28997 }
28998
28999 fn seed_dataset(&self, dataset: DatasetRecord) {
29000 self.state
29001 .datasets_by_version
29002 .lock()
29003 .expect("asset workflow dataset lock should not be poisoned")
29004 .insert(dataset.dataset_id.0, dataset);
29005 }
29006
29007 fn seed_variable(&self, variable: VariableRecord) {
29008 self.state
29009 .variables_by_id
29010 .lock()
29011 .expect("asset workflow variable lock should not be poisoned")
29012 .insert(variable.variable_id.0, variable);
29013 }
29014
29015 fn seed_vocabulary(&self, vocabulary: VocabularyRecord) {
29016 self.state
29017 .vocabularies_by_id
29018 .lock()
29019 .expect("asset workflow vocabulary lock should not be poisoned")
29020 .insert(vocabulary.vocabulary_id.0, vocabulary);
29021 }
29022
29023 fn seed_vocabulary_items(
29024 &self,
29025 vocabulary_id: VocabularyId,
29026 items: Vec<VocabularyItemRecord>,
29027 ) {
29028 self.state
29029 .vocabulary_items_by_vocabulary
29030 .lock()
29031 .expect("asset workflow vocabulary item lock should not be poisoned")
29032 .insert(vocabulary_id.0, items);
29033 }
29034
29035 fn fail_next_save_dataset(&self, message: &str) {
29036 *self
29037 .state
29038 .fail_next_save_dataset
29039 .lock()
29040 .expect("save dataset failure lock should not be poisoned") =
29041 Some(message.to_string());
29042 }
29043
29044 fn fail_next_create_study(&self, message: &str) {
29045 *self
29046 .state
29047 .fail_next_create_study
29048 .lock()
29049 .expect("create study failure lock should not be poisoned") =
29050 Some(message.to_string());
29051 }
29052
29053 fn fail_next_save_datafile(&self, message: &str) {
29054 *self
29055 .state
29056 .fail_next_save_datafile
29057 .lock()
29058 .expect("save datafile failure lock should not be poisoned") =
29059 Some(message.to_string());
29060 }
29061
29062 fn fail_next_save_asset(&self, message: &str) {
29063 *self
29064 .state
29065 .fail_next_save_asset
29066 .lock()
29067 .expect("save asset failure lock should not be poisoned") =
29068 Some(message.to_string());
29069 }
29070
29071 fn fail_next_save_asset_version(&self, message: &str) {
29072 *self
29073 .state
29074 .fail_next_save_asset_version
29075 .lock()
29076 .expect("save asset version failure lock should not be poisoned") =
29077 Some(message.to_string());
29078 }
29079
29080 fn fail_next_list_domain_variables(&self, message: &str) {
29081 *self
29082 .state
29083 .fail_next_list_domain_variables
29084 .lock()
29085 .expect("list domain variables failure lock should not be poisoned") =
29086 Some(message.to_string());
29087 }
29088
29089 fn fail_next_link_dataset_variable(&self, message: &str) {
29090 *self
29091 .state
29092 .fail_next_link_dataset_variable
29093 .lock()
29094 .expect("dataset-variable failure lock should not be poisoned") =
29095 Some(message.to_string());
29096 }
29097
29098 fn fail_next_create_vocabulary(&self, message: &str) {
29099 *self
29100 .state
29101 .fail_next_create_vocabulary
29102 .lock()
29103 .expect("vocabulary failure lock should not be poisoned") =
29104 Some(message.to_string());
29105 }
29106
29107 fn fail_next_attach_tag(&self, message: &str) {
29108 *self
29109 .state
29110 .fail_next_attach_tag
29111 .lock()
29112 .expect("tag attachment failure lock should not be poisoned") =
29113 Some(message.to_string());
29114 }
29115
29116 fn fail_next_transformation_outputs(&self, message: &str) {
29117 *self
29118 .transformations
29119 .state
29120 .fail_next_add_outputs
29121 .lock()
29122 .expect("transformation output failure lock should not be poisoned") =
29123 Some(message.to_string());
29124 }
29125
29126 fn grant_study_access(&self, study_id: StudyId, user_id: &str) {
29127 let mut grants = self
29128 .state
29129 .access_grants
29130 .lock()
29131 .expect("asset workflow access grants lock should not be poisoned");
29132 grants.push(StudyAccessGrantRecord {
29133 study_id,
29134 user_id: user_id.to_string(),
29135 date_granted: None,
29136 granted_by: None,
29137 });
29138 }
29139
29140 fn grant_study_custodianship(&self, study_id: StudyId, user_id: &str) {
29141 let mut custodians = self
29142 .state
29143 .custodians
29144 .lock()
29145 .expect("asset workflow custodians lock should not be poisoned");
29146 custodians.push(StudyCustodianLinkRecord {
29147 study_id,
29148 user_id: user_id.to_string(),
29149 is_primary: false,
29150 date_granted: None,
29151 granted_by: None,
29152 });
29153 }
29154
29155 fn service(&self) -> AppService {
29156 AppService::new(AppRepositories {
29157 registrations: self.domains.clone(),
29158 domains: self.domains.clone(),
29159 studies: self.studies.clone(),
29160 study_domains: self.study_domains.clone(),
29161 study_governance: Arc::new(StatefulAssetStudyGovernanceRepository {
29162 state: self.state.clone(),
29163 }),
29164 assets: self.assets.clone(),
29165 variables: self.variables.clone(),
29166 vocabularies: self.vocabularies.clone(),
29167 entities: Arc::new(NoopEntityRepository),
29168 transformations: self.transformations.clone(),
29169 tags: Arc::new(StatefulTagRepository {
29170 state: self.state.clone(),
29171 }),
29172 })
29173 }
29174 }
29175
29176 struct StatefulAssetStudyGovernanceRepository {
29177 state: Arc<AssetWorkflowState>,
29178 }
29179
29180 struct EmptyEntityRepository;
29181
29182 #[async_trait]
29183 impl EntityRepository for EmptyEntityRepository {
29184 async fn list_entities(
29185 &self,
29186 _domain_id: DomainId,
29187 ) -> Result<Vec<EntityRecord>, ahri_tre_core::CoreError> {
29188 Ok(Vec::new())
29189 }
29190
29191 async fn list_entity_relations(
29192 &self,
29193 _domain_id: DomainId,
29194 ) -> Result<Vec<EntityRelationRecord>, ahri_tre_core::CoreError> {
29195 Ok(Vec::new())
29196 }
29197
29198 async fn create_entity(
29199 &self,
29200 _entity: &NewEntityRecord,
29201 ) -> Result<EntityRecord, ahri_tre_core::CoreError> {
29202 panic!("test repository should not be called")
29203 }
29204
29205 async fn save_entity(
29206 &self,
29207 _entity: &EntityRecord,
29208 ) -> Result<EntityRecord, ahri_tre_core::CoreError> {
29209 panic!("test repository should not be called")
29210 }
29211
29212 async fn create_entity_relation(
29213 &self,
29214 _relation: &NewEntityRelationRecord,
29215 ) -> Result<EntityRelationRecord, ahri_tre_core::CoreError> {
29216 panic!("test repository should not be called")
29217 }
29218
29219 async fn save_entity_relation(
29220 &self,
29221 _relation: &EntityRelationRecord,
29222 ) -> Result<EntityRelationRecord, ahri_tre_core::CoreError> {
29223 panic!("test repository should not be called")
29224 }
29225 }
29226
29227 #[async_trait]
29228 impl StudyGovernanceRepository for StatefulAssetStudyGovernanceRepository {
29229 async fn list_access_grants(
29230 &self,
29231 study_id: StudyId,
29232 ) -> Result<Vec<StudyAccessGrantRecord>, ahri_tre_core::CoreError> {
29233 Ok(self
29234 .state
29235 .access_grants
29236 .lock()
29237 .expect("asset workflow access grants lock should not be poisoned")
29238 .iter()
29239 .filter(|grant| grant.study_id == study_id)
29240 .cloned()
29241 .collect())
29242 }
29243
29244 async fn save_access_grant(
29245 &self,
29246 grant: &StudyAccessGrantRecord,
29247 ) -> Result<StudyAccessGrantRecord, ahri_tre_core::CoreError> {
29248 let mut access_grants = self
29249 .state
29250 .access_grants
29251 .lock()
29252 .expect("asset workflow access grants lock should not be poisoned");
29253 if !access_grants.iter().any(|existing| {
29254 existing.study_id == grant.study_id && existing.user_id == grant.user_id
29255 }) {
29256 access_grants.push(grant.clone());
29257 }
29258 Ok(grant.clone())
29259 }
29260
29261 async fn revoke_access_grant(
29262 &self,
29263 study_id: StudyId,
29264 principal: &str,
29265 ) -> Result<(), ahri_tre_core::CoreError> {
29266 self.revoke_study_access(study_id, principal).await
29267 }
29268
29269 async fn revoke_study_access(
29270 &self,
29271 _study_id: StudyId,
29272 _target_user_id: &str,
29273 ) -> Result<(), ahri_tre_core::CoreError> {
29274 Ok(())
29275 }
29276
29277 async fn list_custodians(
29278 &self,
29279 study_id: StudyId,
29280 ) -> Result<Vec<StudyCustodianLinkRecord>, ahri_tre_core::CoreError> {
29281 Ok(self
29282 .state
29283 .custodians
29284 .lock()
29285 .expect("asset workflow custodians lock should not be poisoned")
29286 .iter()
29287 .filter(|custodian| custodian.study_id == study_id)
29288 .cloned()
29289 .collect())
29290 }
29291
29292 async fn add_delegate_custodian(
29293 &self,
29294 study_id: StudyId,
29295 target_user_id: &str,
29296 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
29297 let custodian = StudyCustodianLinkRecord {
29298 study_id,
29299 user_id: target_user_id.to_string(),
29300 is_primary: false,
29301 date_granted: None,
29302 granted_by: None,
29303 };
29304 self.state
29305 .add_delegate_custodian_calls
29306 .lock()
29307 .expect("delegate custodian call lock should not be poisoned")
29308 .push((study_id, target_user_id.to_string()));
29309 self.save_custodian(&custodian).await
29310 }
29311
29312 async fn transfer_primary_custodianship(
29313 &self,
29314 study_id: StudyId,
29315 target_user_id: &str,
29316 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
29317 self.state
29318 .transfer_primary_custodianship_calls
29319 .lock()
29320 .expect("primary transfer call lock should not be poisoned")
29321 .push((study_id, target_user_id.to_string()));
29322 let mut custodians = self
29323 .state
29324 .custodians
29325 .lock()
29326 .expect("asset workflow custodians lock should not be poisoned");
29327 for custodian in custodians
29328 .iter_mut()
29329 .filter(|custodian| custodian.study_id == study_id)
29330 {
29331 custodian.is_primary = custodian.user_id == target_user_id;
29332 }
29333 if !custodians.iter().any(|custodian| {
29334 custodian.study_id == study_id && custodian.user_id == target_user_id
29335 }) {
29336 custodians.push(StudyCustodianLinkRecord {
29337 study_id,
29338 user_id: target_user_id.to_string(),
29339 is_primary: true,
29340 date_granted: None,
29341 granted_by: None,
29342 });
29343 }
29344 Ok(StudyCustodianLinkRecord {
29345 study_id,
29346 user_id: target_user_id.to_string(),
29347 is_primary: true,
29348 date_granted: None,
29349 granted_by: None,
29350 })
29351 }
29352
29353 async fn remove_delegate_custodian(
29354 &self,
29355 study_id: StudyId,
29356 target_user_id: &str,
29357 ) -> Result<(), ahri_tre_core::CoreError> {
29358 self.state
29359 .remove_delegate_custodian_calls
29360 .lock()
29361 .expect("delegate custodian removal call lock should not be poisoned")
29362 .push((study_id, target_user_id.to_string()));
29363 self.state
29364 .custodians
29365 .lock()
29366 .expect("study custodian lock should not be poisoned")
29367 .retain(|custodian| {
29368 !(custodian.study_id == study_id && custodian.user_id == target_user_id)
29369 });
29370 Ok(())
29371 }
29372
29373 async fn save_custodian(
29374 &self,
29375 custodian: &StudyCustodianLinkRecord,
29376 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
29377 let mut custodians = self
29378 .state
29379 .custodians
29380 .lock()
29381 .expect("asset workflow custodians lock should not be poisoned");
29382 if !custodians.iter().any(|existing| {
29383 existing.study_id == custodian.study_id && existing.user_id == custodian.user_id
29384 }) {
29385 custodians.push(custodian.clone());
29386 }
29387 Ok(custodian.clone())
29388 }
29389
29390 async fn list_study_duo_restrictions(
29391 &self,
29392 study_id: StudyId,
29393 ) -> Result<Vec<StudyDuoRestrictionRecord>, ahri_tre_core::CoreError> {
29394 Ok(self
29395 .state
29396 .study_duo_restrictions
29397 .lock()
29398 .expect("study DUO restriction lock should not be poisoned")
29399 .iter()
29400 .filter(|restriction| restriction.study_id == study_id)
29401 .cloned()
29402 .collect())
29403 }
29404
29405 async fn create_study_duo_restriction(
29406 &self,
29407 restriction: &NewStudyDuoRestrictionRecord,
29408 ) -> Result<StudyDuoRestrictionRecord, ahri_tre_core::CoreError> {
29409 let mut restrictions = self
29410 .state
29411 .study_duo_restrictions
29412 .lock()
29413 .expect("study DUO restriction lock should not be poisoned");
29414 let restriction_id = restrictions
29415 .iter()
29416 .map(|restriction| restriction.study_duo_restriction_id.0)
29417 .max()
29418 .unwrap_or(0)
29419 + 1;
29420 let persisted = StudyDuoRestrictionRecord {
29421 study_duo_restriction_id: StudyDuoRestrictionId(restriction_id),
29422 study_id: restriction.study_id,
29423 duo_id: restriction.duo_id.clone(),
29424 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
29425 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
29426 qualifier_text: restriction.qualifier_text.clone(),
29427 qualifier_date: restriction.qualifier_date,
29428 qualifier_integer: restriction.qualifier_integer,
29429 };
29430 restrictions.push(persisted.clone());
29431 Ok(persisted)
29432 }
29433
29434 async fn save_study_duo_restriction(
29435 &self,
29436 restriction: &StudyDuoRestrictionRecord,
29437 ) -> Result<StudyDuoRestrictionRecord, ahri_tre_core::CoreError> {
29438 let mut restrictions = self
29439 .state
29440 .study_duo_restrictions
29441 .lock()
29442 .expect("study DUO restriction lock should not be poisoned");
29443 if let Some(existing) = restrictions.iter_mut().find(|existing| {
29444 existing.study_duo_restriction_id == restriction.study_duo_restriction_id
29445 }) {
29446 *existing = restriction.clone();
29447 } else {
29448 restrictions.push(restriction.clone());
29449 }
29450 Ok(restriction.clone())
29451 }
29452
29453 async fn replace_study_duo_restrictions(
29454 &self,
29455 study_id: StudyId,
29456 restrictions: &[NewStudyDuoRestrictionRecord],
29457 ) -> Result<Vec<StudyDuoRestrictionRecord>, ahri_tre_core::CoreError> {
29458 let mut persisted = self
29459 .state
29460 .study_duo_restrictions
29461 .lock()
29462 .expect("study DUO restriction lock should not be poisoned");
29463 persisted.retain(|restriction| restriction.study_id != study_id);
29464 let next_id = persisted
29465 .iter()
29466 .map(|restriction| restriction.study_duo_restriction_id.0)
29467 .max()
29468 .unwrap_or(0)
29469 + 1;
29470 let mut replaced = Vec::with_capacity(restrictions.len());
29471 for (offset, restriction) in restrictions.iter().enumerate() {
29472 let row = StudyDuoRestrictionRecord {
29473 study_duo_restriction_id: StudyDuoRestrictionId(next_id + offset as i64),
29474 study_id,
29475 duo_id: restriction.duo_id.clone(),
29476 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
29477 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
29478 qualifier_text: restriction.qualifier_text.clone(),
29479 qualifier_date: restriction.qualifier_date,
29480 qualifier_integer: restriction.qualifier_integer,
29481 };
29482 persisted.push(row.clone());
29483 replaced.push(row);
29484 }
29485 Ok(replaced)
29486 }
29487 }
29488
29489 struct TransformationRepositories {
29490 repository: Arc<StatefulTransformationRepository>,
29491 }
29492
29493 impl TransformationRepositories {
29494 fn new() -> Self {
29495 Self {
29496 repository: Arc::new(StatefulTransformationRepository {
29497 state: Arc::new(TransformationState::default()),
29498 }),
29499 }
29500 }
29501
29502 fn service(&self) -> AppService {
29503 AppService::new(AppRepositories {
29504 registrations: Arc::new(NoopCatalogueRegistrationRepository),
29505 domains: Arc::new(NoopDomainRepository),
29506 studies: Arc::new(NoopStudyRepository),
29507 study_domains: Arc::new(NoopStudyDomainRepository),
29508 study_governance: Arc::new(NoopStudyGovernanceRepository),
29509 assets: Arc::new(NoopAssetRepository),
29510 variables: Arc::new(NoopVariableRepository),
29511 vocabularies: Arc::new(NoopVocabularyRepository),
29512 entities: Arc::new(NoopEntityRepository),
29513 transformations: self.repository.clone(),
29514 tags: Arc::new(NoopTagRepository),
29515 })
29516 }
29517 }
29518
29519 #[derive(Default)]
29520 struct EntityGraphState {
29521 domains_by_id: Mutex<BTreeMap<i64, DomainRecord>>,
29522 studies_by_id: Mutex<BTreeMap<uuid::Uuid, StudyRecord>>,
29523 entities_by_id: Mutex<BTreeMap<i64, EntityRecord>>,
29524 relations_by_id: Mutex<BTreeMap<i64, EntityRelationRecord>>,
29525 entity_instances_by_id: Mutex<BTreeMap<i64, EntityInstanceRecord>>,
29526 study_entity_instances: Mutex<Vec<StudyEntityInstanceRecord>>,
29527 relation_instances_by_id: Mutex<BTreeMap<i64, RelationInstanceRecord>>,
29528 study_relation_instances: Mutex<Vec<StudyRelationInstanceRecord>>,
29529 asset_entity_links: Mutex<Vec<AssetVersionEntityRecord>>,
29530 asset_relation_links: Mutex<Vec<AssetVersionRelationInstanceRecord>>,
29531 dataset_entity_links: Mutex<Vec<DatasetVersionEntityRecord>>,
29532 dataset_relation_links: Mutex<Vec<DatasetVersionRelationInstanceRecord>>,
29533 }
29534
29535 fn next_i64_key<T>(records: &Mutex<BTreeMap<i64, T>>) -> i64 {
29536 records
29537 .lock()
29538 .expect("id assignment lock should not be poisoned")
29539 .keys()
29540 .next_back()
29541 .copied()
29542 .unwrap_or(0)
29543 + 1
29544 }
29545
29546 fn test_uuid(namespace: u128, id: i64) -> uuid::Uuid {
29547 uuid::Uuid::from_u128((namespace << 64) + id as u128)
29548 }
29549
29550 struct StatefulEntityGraphDomainRepository {
29551 state: Arc<EntityGraphState>,
29552 }
29553
29554 #[async_trait]
29555 impl DomainRepository for StatefulEntityGraphDomainRepository {
29556 async fn list_domains(&self) -> Result<Vec<DomainRecord>, ahri_tre_core::CoreError> {
29557 Ok(self
29558 .state
29559 .domains_by_id
29560 .lock()
29561 .expect("entity graph domain lock should not be poisoned")
29562 .values()
29563 .cloned()
29564 .collect())
29565 }
29566
29567 async fn get_domain_by_id(
29568 &self,
29569 domain_id: DomainId,
29570 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
29571 Ok(self
29572 .state
29573 .domains_by_id
29574 .lock()
29575 .expect("entity graph domain lock should not be poisoned")
29576 .get(&domain_id.0)
29577 .cloned())
29578 }
29579
29580 async fn get_domain_by_name(
29581 &self,
29582 name: &str,
29583 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
29584 Ok(self
29585 .state
29586 .domains_by_id
29587 .lock()
29588 .expect("entity graph domain lock should not be poisoned")
29589 .values()
29590 .find(|domain| domain.name.as_str() == name)
29591 .cloned())
29592 }
29593
29594 async fn create_domain(
29595 &self,
29596 domain: &NewDomainRecord,
29597 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
29598 let domain_id = next_i64_key(&self.state.domains_by_id);
29599 let domain = DomainRecord {
29600 domain_id: DomainId(domain_id),
29601 name: domain.name.clone(),
29602 description: domain.description.clone(),
29603 uri: domain.uri.clone(),
29604 };
29605 self.save_domain(&domain).await
29606 }
29607
29608 async fn save_domain(
29609 &self,
29610 domain: &DomainRecord,
29611 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
29612 self.state
29613 .domains_by_id
29614 .lock()
29615 .expect("entity graph domain lock should not be poisoned")
29616 .insert(domain.domain_id.0, domain.clone());
29617 Ok(domain.clone())
29618 }
29619
29620 async fn delete_domain_record(
29621 &self,
29622 domain_id: DomainId,
29623 ) -> Result<bool, ahri_tre_core::CoreError> {
29624 Ok(self
29625 .state
29626 .domains_by_id
29627 .lock()
29628 .expect("domain lock should not be poisoned")
29629 .remove(&domain_id.0)
29630 .is_some())
29631 }
29632 }
29633
29634 struct StatefulEntityGraphStudyRepository {
29635 state: Arc<EntityGraphState>,
29636 }
29637
29638 #[async_trait]
29639 impl StudyRepository for StatefulEntityGraphStudyRepository {
29640 async fn list_studies(&self) -> Result<Vec<StudyRecord>, ahri_tre_core::CoreError> {
29641 Ok(self
29642 .state
29643 .studies_by_id
29644 .lock()
29645 .expect("entity graph study lock should not be poisoned")
29646 .values()
29647 .cloned()
29648 .collect())
29649 }
29650
29651 async fn get_study_by_id(
29652 &self,
29653 study_id: StudyId,
29654 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
29655 Ok(self
29656 .state
29657 .studies_by_id
29658 .lock()
29659 .expect("entity graph study lock should not be poisoned")
29660 .get(&study_id.0)
29661 .cloned())
29662 }
29663
29664 async fn get_study_by_name(
29665 &self,
29666 name: &str,
29667 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
29668 Ok(self
29669 .state
29670 .studies_by_id
29671 .lock()
29672 .expect("entity graph study lock should not be poisoned")
29673 .values()
29674 .find(|study| study.name.as_str() == name)
29675 .cloned())
29676 }
29677
29678 async fn search_studies(
29679 &self,
29680 _query: &ahri_tre_core::StudySearchQuery,
29681 ) -> Result<
29682 ahri_tre_core::SearchPage<StudyRecord, ahri_tre_core::StudySearchCursorAnchor>,
29683 ahri_tre_core::CoreError,
29684 > {
29685 Err(ahri_tre_core::CoreError::NotImplemented("search studies"))
29686 }
29687
29688 async fn create_study(
29689 &self,
29690 study: &NewStudyRecord,
29691 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
29692 let study = StudyRecord {
29693 study_id: StudyId(uuid::Uuid::new_v4()),
29694 name: study.name.clone(),
29695 description: study.description.clone(),
29696 documentation: study.documentation.clone(),
29697 agent_instructions: study.agent_instructions.clone(),
29698 external_id: study.external_id.clone(),
29699 study_type_id: study.study_type_id,
29700 date_created: study.date_created,
29701 created_by: study.created_by.clone(),
29702 };
29703 self.save_study(&study).await
29704 }
29705
29706 async fn save_study(
29707 &self,
29708 study: &StudyRecord,
29709 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
29710 self.state
29711 .studies_by_id
29712 .lock()
29713 .expect("entity graph study lock should not be poisoned")
29714 .insert(study.study_id.0, study.clone());
29715 Ok(study.clone())
29716 }
29717 }
29718
29719 struct StatefulEntityGraphRepository {
29720 state: Arc<EntityGraphState>,
29721 }
29722
29723 #[async_trait]
29724 impl EntityRepository for StatefulEntityGraphRepository {
29725 async fn list_entities(
29726 &self,
29727 domain_id: DomainId,
29728 ) -> Result<Vec<EntityRecord>, ahri_tre_core::CoreError> {
29729 Ok(self
29730 .state
29731 .entities_by_id
29732 .lock()
29733 .expect("entity lock should not be poisoned")
29734 .values()
29735 .filter(|entity| entity.domain_id == domain_id)
29736 .cloned()
29737 .collect())
29738 }
29739
29740 async fn list_entity_relations(
29741 &self,
29742 domain_id: DomainId,
29743 ) -> Result<Vec<EntityRelationRecord>, ahri_tre_core::CoreError> {
29744 Ok(self
29745 .state
29746 .relations_by_id
29747 .lock()
29748 .expect("entity relation lock should not be poisoned")
29749 .values()
29750 .filter(|relation| relation.domain_id == domain_id)
29751 .cloned()
29752 .collect())
29753 }
29754
29755 async fn create_entity(
29756 &self,
29757 entity: &NewEntityRecord,
29758 ) -> Result<EntityRecord, ahri_tre_core::CoreError> {
29759 let entity_id = EntityId(next_i64_key(&self.state.entities_by_id));
29760 let entity = EntityRecord {
29761 entity_id,
29762 domain_id: entity.domain_id,
29763 uuid: Some(test_uuid(0x1000, entity_id.0)),
29764 name: entity.name.clone(),
29765 description: entity.description.clone(),
29766 agent_instructions: entity.agent_instructions.clone(),
29767 ontology_namespace: entity.ontology_namespace.clone(),
29768 ontology_class: entity.ontology_class.clone(),
29769 };
29770 self.save_entity(&entity).await
29771 }
29772
29773 async fn save_entity(
29774 &self,
29775 entity: &EntityRecord,
29776 ) -> Result<EntityRecord, ahri_tre_core::CoreError> {
29777 self.state
29778 .entities_by_id
29779 .lock()
29780 .expect("entity lock should not be poisoned")
29781 .insert(entity.entity_id.0, entity.clone());
29782 Ok(entity.clone())
29783 }
29784
29785 async fn create_entity_relation(
29786 &self,
29787 relation: &NewEntityRelationRecord,
29788 ) -> Result<EntityRelationRecord, ahri_tre_core::CoreError> {
29789 let entity_relation_id = EntityRelationId(next_i64_key(&self.state.relations_by_id));
29790 let relation = EntityRelationRecord {
29791 entity_relation_id,
29792 subject_entity_id: relation.subject_entity_id,
29793 object_entity_id: relation.object_entity_id,
29794 domain_id: relation.domain_id,
29795 uuid: Some(test_uuid(0x2000, entity_relation_id.0)),
29796 name: relation.name.clone(),
29797 description: relation.description.clone(),
29798 agent_instructions: relation.agent_instructions.clone(),
29799 ontology_namespace: relation.ontology_namespace.clone(),
29800 ontology_class: relation.ontology_class.clone(),
29801 };
29802 self.save_entity_relation(&relation).await
29803 }
29804
29805 async fn save_entity_relation(
29806 &self,
29807 relation: &EntityRelationRecord,
29808 ) -> Result<EntityRelationRecord, ahri_tre_core::CoreError> {
29809 self.state
29810 .relations_by_id
29811 .lock()
29812 .expect("entity relation lock should not be poisoned")
29813 .insert(relation.entity_relation_id.0, relation.clone());
29814 Ok(relation.clone())
29815 }
29816
29817 async fn delete_entity_record(
29818 &self,
29819 entity_id: EntityId,
29820 ) -> Result<bool, ahri_tre_core::CoreError> {
29821 Ok(self
29822 .state
29823 .entities_by_id
29824 .lock()
29825 .expect("entity lock should not be poisoned")
29826 .remove(&entity_id.0)
29827 .is_some())
29828 }
29829
29830 async fn delete_entity_relation_record(
29831 &self,
29832 relation_id: EntityRelationId,
29833 ) -> Result<bool, ahri_tre_core::CoreError> {
29834 Ok(self
29835 .state
29836 .relations_by_id
29837 .lock()
29838 .expect("entity relation lock should not be poisoned")
29839 .remove(&relation_id.0)
29840 .is_some())
29841 }
29842
29843 async fn get_entity_instance(
29844 &self,
29845 instance_id: i64,
29846 ) -> Result<Option<EntityInstanceRecord>, ahri_tre_core::CoreError> {
29847 Ok(self
29848 .state
29849 .entity_instances_by_id
29850 .lock()
29851 .expect("entity instance lock should not be poisoned")
29852 .get(&instance_id)
29853 .cloned())
29854 }
29855
29856 async fn get_entity_instance_by_uuid(
29857 &self,
29858 uuid: uuid::Uuid,
29859 ) -> Result<Option<EntityInstanceRecord>, ahri_tre_core::CoreError> {
29860 Ok(self
29861 .state
29862 .entity_instances_by_id
29863 .lock()
29864 .expect("entity instance lock should not be poisoned")
29865 .values()
29866 .find(|instance| instance.uuid == uuid)
29867 .cloned())
29868 }
29869
29870 async fn list_entity_instances(
29871 &self,
29872 entity_id: EntityId,
29873 ) -> Result<Vec<EntityInstanceRecord>, ahri_tre_core::CoreError> {
29874 Ok(self
29875 .state
29876 .entity_instances_by_id
29877 .lock()
29878 .expect("entity instance lock should not be poisoned")
29879 .values()
29880 .filter(|instance| instance.entity_id == entity_id)
29881 .cloned()
29882 .collect())
29883 }
29884
29885 async fn save_entity_instance(
29886 &self,
29887 instance: &EntityInstanceWriteRecord,
29888 ) -> Result<EntityInstanceRecord, ahri_tre_core::CoreError> {
29889 let instance_id = instance
29890 .instance_id
29891 .unwrap_or_else(|| next_i64_key(&self.state.entity_instances_by_id));
29892 let instance = EntityInstanceRecord {
29893 instance_id,
29894 uuid: instance
29895 .uuid
29896 .unwrap_or_else(|| test_uuid(0x3000, instance_id)),
29897 entity_id: instance.entity_id,
29898 label: instance.label.clone(),
29899 note: instance.note.clone(),
29900 transformation_id: instance.transformation_id,
29901 };
29902 self.state
29903 .entity_instances_by_id
29904 .lock()
29905 .expect("entity instance lock should not be poisoned")
29906 .insert(instance_id, instance.clone());
29907 Ok(instance)
29908 }
29909
29910 async fn create_entity_instance(
29911 &self,
29912 instance: &NewEntityInstanceRecord,
29913 ) -> Result<EntityInstanceRecord, ahri_tre_core::CoreError> {
29914 self.save_entity_instance(&EntityInstanceWriteRecord {
29915 instance_id: None,
29916 uuid: None,
29917 entity_id: instance.entity_id,
29918 label: instance.label.clone(),
29919 note: instance.note.clone(),
29920 transformation_id: instance.transformation_id,
29921 })
29922 .await
29923 }
29924
29925 async fn get_study_entity_instance(
29926 &self,
29927 study_id: StudyId,
29928 entity_id: EntityId,
29929 external_id: &str,
29930 ) -> Result<Option<StudyEntityInstanceRecord>, ahri_tre_core::CoreError> {
29931 Ok(self
29932 .state
29933 .study_entity_instances
29934 .lock()
29935 .expect("study entity instance lock should not be poisoned")
29936 .iter()
29937 .find(|mapping| {
29938 mapping.study_id == study_id
29939 && mapping.entity_id == entity_id
29940 && mapping.external_id == external_id
29941 })
29942 .cloned())
29943 }
29944
29945 async fn save_study_entity_instance(
29946 &self,
29947 mapping: &ahri_tre_types::StudyEntityInstanceWriteRecord,
29948 ) -> Result<StudyEntityInstanceRecord, ahri_tre_core::CoreError> {
29949 let mapping = StudyEntityInstanceRecord {
29950 study_id: mapping.study_id,
29951 entity_instance_id: mapping.entity_instance_id,
29952 entity_id: mapping.entity_id,
29953 external_id: mapping.external_id.clone(),
29954 transformation_id: mapping.transformation_id,
29955 };
29956 let mut mappings = self
29957 .state
29958 .study_entity_instances
29959 .lock()
29960 .expect("study entity instance lock should not be poisoned");
29961 if let Some(existing) = mappings.iter_mut().find(|existing| {
29962 existing.study_id == mapping.study_id
29963 && existing.entity_instance_id == mapping.entity_instance_id
29964 }) {
29965 *existing = mapping.clone();
29966 } else {
29967 mappings.push(mapping.clone());
29968 }
29969 Ok(mapping)
29970 }
29971
29972 async fn list_study_entity_instances(
29973 &self,
29974 study_id: StudyId,
29975 ) -> Result<Vec<StudyEntityInstanceRecord>, ahri_tre_core::CoreError> {
29976 Ok(self
29977 .state
29978 .study_entity_instances
29979 .lock()
29980 .expect("study entity instance lock should not be poisoned")
29981 .iter()
29982 .filter(|mapping| mapping.study_id == study_id)
29983 .cloned()
29984 .collect())
29985 }
29986
29987 async fn get_relation_instance(
29988 &self,
29989 relation_instance_id: i64,
29990 ) -> Result<Option<RelationInstanceRecord>, ahri_tre_core::CoreError> {
29991 Ok(self
29992 .state
29993 .relation_instances_by_id
29994 .lock()
29995 .expect("relation instance lock should not be poisoned")
29996 .get(&relation_instance_id)
29997 .cloned())
29998 }
29999
30000 async fn list_relation_instances(
30001 &self,
30002 relation_id: EntityRelationId,
30003 ) -> Result<Vec<RelationInstanceRecord>, ahri_tre_core::CoreError> {
30004 Ok(self
30005 .state
30006 .relation_instances_by_id
30007 .lock()
30008 .expect("relation instance lock should not be poisoned")
30009 .values()
30010 .filter(|instance| instance.entity_relation_id == relation_id)
30011 .cloned()
30012 .collect())
30013 }
30014
30015 async fn save_relation_instance(
30016 &self,
30017 relation: &RelationInstanceWriteRecord,
30018 ) -> Result<RelationInstanceRecord, ahri_tre_core::CoreError> {
30019 let relation_instance_id = relation
30020 .relation_instance_id
30021 .unwrap_or_else(|| next_i64_key(&self.state.relation_instances_by_id));
30022 let relation = RelationInstanceRecord {
30023 relation_instance_id,
30024 uuid: relation
30025 .uuid
30026 .unwrap_or_else(|| test_uuid(0x4000, relation_instance_id)),
30027 entity_relation_id: relation.entity_relation_id,
30028 entity_instance_id_1: relation.entity_instance_id_1,
30029 entity_instance_id_2: relation.entity_instance_id_2,
30030 valid_from: relation.valid_from,
30031 valid_to: relation.valid_to,
30032 note: relation.note.clone(),
30033 transformation_id: relation.transformation_id,
30034 };
30035 self.state
30036 .relation_instances_by_id
30037 .lock()
30038 .expect("relation instance lock should not be poisoned")
30039 .insert(relation_instance_id, relation.clone());
30040 Ok(relation)
30041 }
30042
30043 async fn create_relation_instance(
30044 &self,
30045 relation: &NewRelationInstanceRecord,
30046 ) -> Result<RelationInstanceRecord, ahri_tre_core::CoreError> {
30047 self.save_relation_instance(&RelationInstanceWriteRecord {
30048 relation_instance_id: None,
30049 uuid: None,
30050 entity_relation_id: relation.entity_relation_id,
30051 entity_instance_id_1: relation.entity_instance_id_1,
30052 entity_instance_id_2: relation.entity_instance_id_2,
30053 valid_from: relation.valid_from,
30054 valid_to: relation.valid_to,
30055 note: relation.note.clone(),
30056 transformation_id: relation.transformation_id,
30057 })
30058 .await
30059 }
30060
30061 async fn get_study_relation_instance(
30062 &self,
30063 study_id: StudyId,
30064 relation_id: EntityRelationId,
30065 external_id: &str,
30066 ) -> Result<Option<StudyRelationInstanceRecord>, ahri_tre_core::CoreError> {
30067 Ok(self
30068 .state
30069 .study_relation_instances
30070 .lock()
30071 .expect("study relation instance lock should not be poisoned")
30072 .iter()
30073 .find(|mapping| {
30074 mapping.study_id == study_id
30075 && mapping.entity_relation_id == relation_id
30076 && mapping.external_id == external_id
30077 })
30078 .cloned())
30079 }
30080
30081 async fn save_study_relation_instance(
30082 &self,
30083 mapping: &ahri_tre_types::StudyRelationInstanceWriteRecord,
30084 ) -> Result<StudyRelationInstanceRecord, ahri_tre_core::CoreError> {
30085 let mapping = StudyRelationInstanceRecord {
30086 study_id: mapping.study_id,
30087 relation_instance_id: mapping.relation_instance_id,
30088 entity_relation_id: mapping.entity_relation_id,
30089 external_id: mapping.external_id.clone(),
30090 transformation_id: mapping.transformation_id,
30091 };
30092 let mut mappings = self
30093 .state
30094 .study_relation_instances
30095 .lock()
30096 .expect("study relation instance lock should not be poisoned");
30097 if let Some(existing) = mappings.iter_mut().find(|existing| {
30098 existing.study_id == mapping.study_id
30099 && existing.relation_instance_id == mapping.relation_instance_id
30100 }) {
30101 *existing = mapping.clone();
30102 } else {
30103 mappings.push(mapping.clone());
30104 }
30105 Ok(mapping)
30106 }
30107
30108 async fn list_study_relation_instances(
30109 &self,
30110 study_id: StudyId,
30111 ) -> Result<Vec<StudyRelationInstanceRecord>, ahri_tre_core::CoreError> {
30112 Ok(self
30113 .state
30114 .study_relation_instances
30115 .lock()
30116 .expect("study relation instance lock should not be poisoned")
30117 .iter()
30118 .filter(|mapping| mapping.study_id == study_id)
30119 .cloned()
30120 .collect())
30121 }
30122
30123 async fn link_asset_version_entity(
30124 &self,
30125 link: &AssetVersionEntityRecord,
30126 ) -> Result<AssetVersionEntityRecord, ahri_tre_core::CoreError> {
30127 let mut links = self
30128 .state
30129 .asset_entity_links
30130 .lock()
30131 .expect("asset entity link lock should not be poisoned");
30132 if let Some(existing) = links.iter_mut().find(|existing| {
30133 existing.version_id == link.version_id
30134 && existing.entity_instance_id == link.entity_instance_id
30135 }) {
30136 *existing = link.clone();
30137 } else {
30138 links.push(link.clone());
30139 }
30140 Ok(link.clone())
30141 }
30142
30143 async fn list_asset_version_entities(
30144 &self,
30145 version_id: VersionId,
30146 ) -> Result<Vec<AssetVersionEntityRecord>, ahri_tre_core::CoreError> {
30147 Ok(self
30148 .state
30149 .asset_entity_links
30150 .lock()
30151 .expect("asset entity link lock should not be poisoned")
30152 .iter()
30153 .filter(|link| link.version_id == version_id)
30154 .cloned()
30155 .collect())
30156 }
30157
30158 async fn list_asset_version_entities_for_instance(
30159 &self,
30160 entity_instance_id: i64,
30161 ) -> Result<Vec<AssetVersionEntityRecord>, ahri_tre_core::CoreError> {
30162 Ok(self
30163 .state
30164 .asset_entity_links
30165 .lock()
30166 .expect("asset entity link lock should not be poisoned")
30167 .iter()
30168 .filter(|link| link.entity_instance_id == entity_instance_id)
30169 .cloned()
30170 .collect())
30171 }
30172
30173 async fn link_asset_version_relation_instance(
30174 &self,
30175 link: &AssetVersionRelationInstanceRecord,
30176 ) -> Result<AssetVersionRelationInstanceRecord, ahri_tre_core::CoreError> {
30177 let mut links = self
30178 .state
30179 .asset_relation_links
30180 .lock()
30181 .expect("asset relation link lock should not be poisoned");
30182 if let Some(existing) = links.iter_mut().find(|existing| {
30183 existing.version_id == link.version_id
30184 && existing.relation_instance_id == link.relation_instance_id
30185 }) {
30186 *existing = link.clone();
30187 } else {
30188 links.push(link.clone());
30189 }
30190 Ok(link.clone())
30191 }
30192
30193 async fn list_asset_version_relation_instances(
30194 &self,
30195 version_id: VersionId,
30196 ) -> Result<Vec<AssetVersionRelationInstanceRecord>, ahri_tre_core::CoreError> {
30197 Ok(self
30198 .state
30199 .asset_relation_links
30200 .lock()
30201 .expect("asset relation link lock should not be poisoned")
30202 .iter()
30203 .filter(|link| link.version_id == version_id)
30204 .cloned()
30205 .collect())
30206 }
30207
30208 async fn list_asset_version_relation_instances_for_instance(
30209 &self,
30210 relation_instance_id: i64,
30211 ) -> Result<Vec<AssetVersionRelationInstanceRecord>, ahri_tre_core::CoreError> {
30212 Ok(self
30213 .state
30214 .asset_relation_links
30215 .lock()
30216 .expect("asset relation link lock should not be poisoned")
30217 .iter()
30218 .filter(|link| link.relation_instance_id == relation_instance_id)
30219 .cloned()
30220 .collect())
30221 }
30222
30223 async fn link_dataset_version_entity(
30224 &self,
30225 link: &DatasetVersionEntityRecord,
30226 ) -> Result<DatasetVersionEntityRecord, ahri_tre_core::CoreError> {
30227 let mut links = self
30228 .state
30229 .dataset_entity_links
30230 .lock()
30231 .expect("dataset entity link lock should not be poisoned");
30232 if let Some(existing) = links.iter_mut().find(|existing| {
30233 existing.version_id == link.version_id
30234 && existing.entity_instance_id == link.entity_instance_id
30235 }) {
30236 *existing = link.clone();
30237 } else {
30238 links.push(link.clone());
30239 }
30240 Ok(link.clone())
30241 }
30242
30243 async fn list_dataset_version_entities(
30244 &self,
30245 version_id: VersionId,
30246 ) -> Result<Vec<DatasetVersionEntityRecord>, ahri_tre_core::CoreError> {
30247 Ok(self
30248 .state
30249 .dataset_entity_links
30250 .lock()
30251 .expect("dataset entity link lock should not be poisoned")
30252 .iter()
30253 .filter(|link| link.version_id == version_id)
30254 .cloned()
30255 .collect())
30256 }
30257
30258 async fn list_dataset_version_entities_for_instance(
30259 &self,
30260 entity_instance_id: i64,
30261 ) -> Result<Vec<DatasetVersionEntityRecord>, ahri_tre_core::CoreError> {
30262 Ok(self
30263 .state
30264 .dataset_entity_links
30265 .lock()
30266 .expect("dataset entity link lock should not be poisoned")
30267 .iter()
30268 .filter(|link| link.entity_instance_id == entity_instance_id)
30269 .cloned()
30270 .collect())
30271 }
30272
30273 async fn link_dataset_version_relation_instance(
30274 &self,
30275 link: &DatasetVersionRelationInstanceRecord,
30276 ) -> Result<DatasetVersionRelationInstanceRecord, ahri_tre_core::CoreError> {
30277 let mut links = self
30278 .state
30279 .dataset_relation_links
30280 .lock()
30281 .expect("dataset relation link lock should not be poisoned");
30282 if let Some(existing) = links.iter_mut().find(|existing| {
30283 existing.version_id == link.version_id
30284 && existing.relation_instance_id == link.relation_instance_id
30285 }) {
30286 *existing = link.clone();
30287 } else {
30288 links.push(link.clone());
30289 }
30290 Ok(link.clone())
30291 }
30292
30293 async fn list_dataset_version_relation_instances(
30294 &self,
30295 version_id: VersionId,
30296 ) -> Result<Vec<DatasetVersionRelationInstanceRecord>, ahri_tre_core::CoreError> {
30297 Ok(self
30298 .state
30299 .dataset_relation_links
30300 .lock()
30301 .expect("dataset relation link lock should not be poisoned")
30302 .iter()
30303 .filter(|link| link.version_id == version_id)
30304 .cloned()
30305 .collect())
30306 }
30307
30308 async fn link_dataset_version_entities(
30309 &self,
30310 links: &[DatasetVersionEntityRecord],
30311 ) -> Result<(), ahri_tre_core::CoreError> {
30312 let mut existing_links = self
30313 .state
30314 .dataset_entity_links
30315 .lock()
30316 .expect("dataset entity link lock should not be poisoned");
30317 for link in links {
30318 if let Some(existing) = existing_links.iter_mut().find(|existing| {
30319 existing.version_id == link.version_id
30320 && existing.entity_instance_id == link.entity_instance_id
30321 }) {
30322 *existing = link.clone();
30323 } else {
30324 existing_links.push(link.clone());
30325 }
30326 }
30327 Ok(())
30328 }
30329
30330 async fn link_dataset_version_relation_instances(
30331 &self,
30332 links: &[DatasetVersionRelationInstanceRecord],
30333 ) -> Result<(), ahri_tre_core::CoreError> {
30334 let mut existing_links = self
30335 .state
30336 .dataset_relation_links
30337 .lock()
30338 .expect("dataset relation link lock should not be poisoned");
30339 for link in links {
30340 if let Some(existing) = existing_links.iter_mut().find(|existing| {
30341 existing.version_id == link.version_id
30342 && existing.relation_instance_id == link.relation_instance_id
30343 }) {
30344 *existing = link.clone();
30345 } else {
30346 existing_links.push(link.clone());
30347 }
30348 }
30349 Ok(())
30350 }
30351 }
30352
30353 struct RegistrationState {
30354 seed_governance_defaults: bool,
30355 domains_by_id: Mutex<BTreeMap<i64, DomainRecord>>,
30356 domains_by_name: Mutex<BTreeMap<String, DomainRecord>>,
30357 studies_by_id: Mutex<BTreeMap<uuid::Uuid, StudyRecord>>,
30358 studies_by_name: Mutex<BTreeMap<String, StudyRecord>>,
30359 study_domains: Mutex<Vec<StudyDomainLinkRecord>>,
30360 tags: Mutex<Vec<ahri_tre_types::TagRecord>>,
30361 tag_links: Mutex<Vec<(TagAttachmentTarget, TagId)>>,
30362 fail_next_registration: Mutex<Option<RegistrationFailurePoint>>,
30363 access_grants: Mutex<Vec<StudyAccessGrantRecord>>,
30364 grant_access_calls: Mutex<Vec<(StudyId, String)>>,
30365 revoke_access_calls: Mutex<Vec<(StudyId, String)>>,
30366 add_delegate_custodian_calls: Mutex<Vec<(StudyId, String)>>,
30367 transfer_primary_custodianship_calls: Mutex<Vec<(StudyId, String)>>,
30368 remove_delegate_custodian_calls: Mutex<Vec<(StudyId, String)>>,
30369 custodians: Mutex<Vec<StudyCustodianLinkRecord>>,
30370 study_duo_restrictions: Mutex<Vec<StudyDuoRestrictionRecord>>,
30371 }
30372
30373 fn prepare_registration_tags(
30374 tags: &mut Vec<ahri_tre_types::TagRecord>,
30375 tag_links: &mut Vec<(TagAttachmentTarget, TagId)>,
30376 target: TagAttachmentTarget,
30377 normalized_tags: &[String],
30378 ) {
30379 for name in normalized_tags {
30380 let tag = match tags.iter().find(|tag| tag.name == *name) {
30381 Some(tag) => tag.clone(),
30382 None => {
30383 let tag = ahri_tre_types::TagRecord {
30384 tag_id: TagId(tags.len() as i64 + 1),
30385 name: name.clone(),
30386 };
30387 tags.push(tag.clone());
30388 tag
30389 }
30390 };
30391 if !tag_links.contains(&(target.clone(), tag.tag_id)) {
30392 tag_links.push((target.clone(), tag.tag_id));
30393 }
30394 }
30395 }
30396
30397 impl RegistrationState {
30398 fn new(seed_governance_defaults: bool) -> Self {
30399 Self {
30400 seed_governance_defaults,
30401 domains_by_id: Mutex::new(BTreeMap::new()),
30402 domains_by_name: Mutex::new(BTreeMap::new()),
30403 studies_by_id: Mutex::new(BTreeMap::new()),
30404 studies_by_name: Mutex::new(BTreeMap::new()),
30405 study_domains: Mutex::new(Vec::new()),
30406 tags: Mutex::new(Vec::new()),
30407 tag_links: Mutex::new(Vec::new()),
30408 fail_next_registration: Mutex::new(None),
30409 access_grants: Mutex::new(Vec::new()),
30410 grant_access_calls: Mutex::new(Vec::new()),
30411 revoke_access_calls: Mutex::new(Vec::new()),
30412 add_delegate_custodian_calls: Mutex::new(Vec::new()),
30413 transfer_primary_custodianship_calls: Mutex::new(Vec::new()),
30414 remove_delegate_custodian_calls: Mutex::new(Vec::new()),
30415 custodians: Mutex::new(Vec::new()),
30416 study_duo_restrictions: Mutex::new(Vec::new()),
30417 }
30418 }
30419
30420 fn fail_registration_at(&self, point: RegistrationFailurePoint) -> Result<(), CoreError> {
30421 let mut failure = self
30422 .fail_next_registration
30423 .lock()
30424 .expect("registration failure lock should not be poisoned");
30425 if failure.as_ref() == Some(&point) {
30426 failure.take();
30427 return Err(CoreError::Infrastructure(format!(
30428 "injected {point:?} registration failure"
30429 )));
30430 }
30431 Ok(())
30432 }
30433
30434 fn registration_tag(&self, name: &str) -> ahri_tre_types::TagRecord {
30435 let mut tags = self.tags.lock().expect("tag lock should not be poisoned");
30436 if let Some(tag) = tags.iter().find(|tag| tag.name == name) {
30437 return tag.clone();
30438 }
30439 let tag = ahri_tre_types::TagRecord {
30440 tag_id: TagId(tags.len() as i64 + 1),
30441 name: name.to_string(),
30442 };
30443 tags.push(tag.clone());
30444 tag
30445 }
30446 }
30447
30448 #[derive(Default)]
30449 struct AssetWorkflowState {
30450 domains_by_id: Mutex<BTreeMap<i64, DomainRecord>>,
30451 studies_by_id: Mutex<BTreeMap<uuid::Uuid, StudyRecord>>,
30452 assets_by_id: Mutex<BTreeMap<uuid::Uuid, AssetRecord>>,
30453 versions_by_asset: Mutex<BTreeMap<uuid::Uuid, Vec<AssetVersionRecord>>>,
30454 datasets_by_version: Mutex<BTreeMap<uuid::Uuid, DatasetRecord>>,
30455 dataset_withdrawals_by_version:
30456 Mutex<BTreeMap<uuid::Uuid, ahri_tre_types::DatasetVersionWithdrawalRecord>>,
30457 datafiles_by_version: Mutex<BTreeMap<uuid::Uuid, DataFileRecord>>,
30458 variables_by_id: Mutex<BTreeMap<i64, VariableRecord>>,
30459 vocabularies_by_id: Mutex<BTreeMap<i64, VocabularyRecord>>,
30460 vocabulary_items_by_vocabulary: Mutex<BTreeMap<i64, Vec<VocabularyItemRecord>>>,
30461 vocabulary_mappings: Mutex<Vec<ahri_tre_types::VocabularyMappingRecord>>,
30462 dataset_variables: Mutex<Vec<ahri_tre_types::DatasetVariableLinkRecord>>,
30463 domains_by_study: Mutex<Vec<(StudyId, DomainRecord)>>,
30464 tags: Mutex<Vec<ahri_tre_types::TagRecord>>,
30465 tag_links: Mutex<Vec<(TagAttachmentTarget, ahri_tre_types::TagId)>>,
30466 access_grants: Mutex<Vec<StudyAccessGrantRecord>>,
30467 add_delegate_custodian_calls: Mutex<Vec<(StudyId, String)>>,
30468 transfer_primary_custodianship_calls: Mutex<Vec<(StudyId, String)>>,
30469 remove_delegate_custodian_calls: Mutex<Vec<(StudyId, String)>>,
30470 custodians: Mutex<Vec<StudyCustodianLinkRecord>>,
30471 study_duo_restrictions: Mutex<Vec<StudyDuoRestrictionRecord>>,
30472 asset_version_duo_restrictions: Mutex<Vec<AssetVersionDuoRestrictionRecord>>,
30473 fail_next_create_study: Mutex<Option<String>>,
30474 fail_next_save_dataset: Mutex<Option<String>>,
30475 fail_next_save_datafile: Mutex<Option<String>>,
30476 fail_next_save_asset: Mutex<Option<String>>,
30477 fail_next_save_asset_version: Mutex<Option<String>>,
30478 fail_next_list_domain_variables: Mutex<Option<String>>,
30479 fail_next_link_dataset_variable: Mutex<Option<String>>,
30480 fail_next_create_vocabulary: Mutex<Option<String>>,
30481 fail_next_attach_tag: Mutex<Option<String>>,
30482 }
30483
30484 struct StatefulAssetDomainRepository {
30485 state: Arc<AssetWorkflowState>,
30486 }
30487
30488 #[async_trait]
30489 impl DomainRepository for StatefulAssetDomainRepository {
30490 async fn list_domains(&self) -> Result<Vec<DomainRecord>, ahri_tre_core::CoreError> {
30491 Ok(self
30492 .state
30493 .domains_by_id
30494 .lock()
30495 .expect("domain lock should not be poisoned")
30496 .values()
30497 .cloned()
30498 .collect())
30499 }
30500
30501 async fn get_domain_by_id(
30502 &self,
30503 domain_id: DomainId,
30504 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
30505 Ok(self
30506 .state
30507 .domains_by_id
30508 .lock()
30509 .expect("domain lock should not be poisoned")
30510 .get(&domain_id.0)
30511 .cloned())
30512 }
30513
30514 async fn get_domain_by_name(
30515 &self,
30516 name: &str,
30517 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
30518 Ok(self
30519 .state
30520 .domains_by_id
30521 .lock()
30522 .expect("domain lock should not be poisoned")
30523 .values()
30524 .find(|domain| domain.name.as_str() == name)
30525 .cloned())
30526 }
30527
30528 async fn create_domain(
30529 &self,
30530 domain: &NewDomainRecord,
30531 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
30532 let domain_id = DomainId(
30533 self.state
30534 .domains_by_id
30535 .lock()
30536 .expect("domain lock should not be poisoned")
30537 .keys()
30538 .max()
30539 .copied()
30540 .unwrap_or(0)
30541 + 1,
30542 );
30543 let domain = DomainRecord {
30544 domain_id,
30545 name: domain.name.clone(),
30546 uri: domain.uri.clone(),
30547 description: domain.description.clone(),
30548 };
30549 self.save_domain(&domain).await
30550 }
30551
30552 async fn save_domain(
30553 &self,
30554 domain: &DomainRecord,
30555 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
30556 self.state
30557 .domains_by_id
30558 .lock()
30559 .expect("domain lock should not be poisoned")
30560 .insert(domain.domain_id.0, domain.clone());
30561 Ok(domain.clone())
30562 }
30563
30564 async fn delete_domain_record(
30565 &self,
30566 domain_id: DomainId,
30567 ) -> Result<bool, ahri_tre_core::CoreError> {
30568 Ok(self
30569 .state
30570 .domains_by_id
30571 .lock()
30572 .expect("domain lock should not be poisoned")
30573 .remove(&domain_id.0)
30574 .is_some())
30575 }
30576 }
30577
30578 #[async_trait]
30579 impl CatalogueRegistrationRepository for StatefulAssetDomainRepository {
30580 async fn create_domain_registration(
30581 &self,
30582 domain: &NewDomainRecord,
30583 normalized_tags: &[String],
30584 ) -> Result<DomainRecord, CoreError> {
30585 let domain = self.create_domain(domain).await?;
30586 attach_asset_workflow_tags(
30587 &self.state,
30588 TagAttachmentTarget::Domain(domain.domain_id),
30589 normalized_tags,
30590 );
30591 Ok(domain)
30592 }
30593
30594 async fn create_study_registration(
30595 &self,
30596 study: &NewStudyRecord,
30597 domain_ids: &[DomainId],
30598 normalized_tags: &[String],
30599 ) -> Result<StudyRecord, CoreError> {
30600 if let Some(message) = self
30601 .state
30602 .fail_next_create_study
30603 .lock()
30604 .expect("create study failure lock should not be poisoned")
30605 .take()
30606 {
30607 return Err(CoreError::Infrastructure(message));
30608 }
30609
30610 let creator = study
30611 .created_by
30612 .clone()
30613 .unwrap_or_else(|| "test_current_user".to_string());
30614 let study = StudyRecord {
30615 study_id: StudyId(uuid::Uuid::new_v4()),
30616 name: study.name.clone(),
30617 description: study.description.clone(),
30618 documentation: study.documentation.clone(),
30619 agent_instructions: study.agent_instructions.clone(),
30620 external_id: study.external_id.clone(),
30621 study_type_id: study.study_type_id,
30622 date_created: study.date_created,
30623 created_by: Some(creator.clone()),
30624 };
30625 let domains = {
30626 let available = self
30627 .state
30628 .domains_by_id
30629 .lock()
30630 .expect("domain lock should not be poisoned");
30631 domain_ids
30632 .iter()
30633 .filter_map(|domain_id| available.get(&domain_id.0).cloned())
30634 .collect::<Vec<_>>()
30635 };
30636
30637 self.state
30638 .studies_by_id
30639 .lock()
30640 .expect("asset workflow study lock should not be poisoned")
30641 .insert(study.study_id.0, study.clone());
30642 self.state
30643 .domains_by_study
30644 .lock()
30645 .expect("asset workflow study-domain lock should not be poisoned")
30646 .extend(domains.into_iter().map(|domain| (study.study_id, domain)));
30647 attach_asset_workflow_tags(
30648 &self.state,
30649 TagAttachmentTarget::Study(study.study_id),
30650 normalized_tags,
30651 );
30652 self.state
30653 .access_grants
30654 .lock()
30655 .expect("study access lock should not be poisoned")
30656 .push(StudyAccessGrantRecord {
30657 study_id: study.study_id,
30658 user_id: creator.clone(),
30659 date_granted: None,
30660 granted_by: Some(creator.clone()),
30661 });
30662 self.state
30663 .custodians
30664 .lock()
30665 .expect("study custodian lock should not be poisoned")
30666 .push(StudyCustodianLinkRecord {
30667 study_id: study.study_id,
30668 user_id: creator.clone(),
30669 is_primary: true,
30670 date_granted: None,
30671 granted_by: Some(creator),
30672 });
30673 Ok(study)
30674 }
30675
30676 async fn save_domain_registration(
30677 &self,
30678 domain: &DomainRecord,
30679 normalized_tags: &[String],
30680 ) -> Result<DomainRecord, CoreError> {
30681 let domain = self.save_domain(domain).await?;
30682 attach_asset_workflow_tags(
30683 &self.state,
30684 TagAttachmentTarget::Domain(domain.domain_id),
30685 normalized_tags,
30686 );
30687 Ok(domain)
30688 }
30689
30690 async fn save_study_registration(
30691 &self,
30692 study: &StudyRecord,
30693 domain_ids: &[DomainId],
30694 normalized_tags: &[String],
30695 ) -> Result<StudyRecord, CoreError> {
30696 let creator = study
30697 .created_by
30698 .clone()
30699 .unwrap_or_else(|| "test_current_user".to_string());
30700 let domains = {
30701 let available = self
30702 .state
30703 .domains_by_id
30704 .lock()
30705 .expect("domain lock should not be poisoned");
30706 domain_ids
30707 .iter()
30708 .filter_map(|domain_id| available.get(&domain_id.0).cloned())
30709 .collect::<Vec<_>>()
30710 };
30711 self.state
30712 .studies_by_id
30713 .lock()
30714 .expect("asset workflow study lock should not be poisoned")
30715 .insert(study.study_id.0, study.clone());
30716 self.state
30717 .domains_by_study
30718 .lock()
30719 .expect("asset workflow study-domain lock should not be poisoned")
30720 .extend(domains.into_iter().map(|domain| (study.study_id, domain)));
30721 attach_asset_workflow_tags(
30722 &self.state,
30723 TagAttachmentTarget::Study(study.study_id),
30724 normalized_tags,
30725 );
30726 self.state
30727 .access_grants
30728 .lock()
30729 .expect("study access lock should not be poisoned")
30730 .push(StudyAccessGrantRecord {
30731 study_id: study.study_id,
30732 user_id: creator.clone(),
30733 date_granted: None,
30734 granted_by: Some(creator.clone()),
30735 });
30736 self.state
30737 .custodians
30738 .lock()
30739 .expect("study custodian lock should not be poisoned")
30740 .push(StudyCustodianLinkRecord {
30741 study_id: study.study_id,
30742 user_id: creator.clone(),
30743 is_primary: true,
30744 date_granted: None,
30745 granted_by: Some(creator),
30746 });
30747 Ok(study.clone())
30748 }
30749 }
30750
30751 fn attach_asset_workflow_tags(
30752 state: &AssetWorkflowState,
30753 target: TagAttachmentTarget,
30754 normalized_tags: &[String],
30755 ) {
30756 for name in normalized_tags {
30757 let tag = {
30758 let mut tags = state.tags.lock().expect("tag lock should not be poisoned");
30759 match tags.iter().find(|tag| tag.name == *name) {
30760 Some(tag) => tag.clone(),
30761 None => {
30762 let tag = ahri_tre_types::TagRecord {
30763 tag_id: TagId(tags.len() as i64 + 1),
30764 name: name.clone(),
30765 };
30766 tags.push(tag.clone());
30767 tag
30768 }
30769 }
30770 };
30771 let mut links = state
30772 .tag_links
30773 .lock()
30774 .expect("tag link lock should not be poisoned");
30775 if !links.contains(&(target.clone(), tag.tag_id)) {
30776 links.push((target.clone(), tag.tag_id));
30777 }
30778 }
30779 }
30780
30781 struct StatefulAssetStudyDomainRepository {
30782 state: Arc<AssetWorkflowState>,
30783 }
30784
30785 #[async_trait]
30786 impl StudyDomainRepository for StatefulAssetStudyDomainRepository {
30787 async fn link_study_domain(
30788 &self,
30789 study_id: StudyId,
30790 domain_id: DomainId,
30791 ) -> Result<StudyDomainLinkRecord, ahri_tre_core::CoreError> {
30792 if let Some(domain) = self
30793 .state
30794 .domains_by_id
30795 .lock()
30796 .expect("domain lock should not be poisoned")
30797 .get(&domain_id.0)
30798 .cloned()
30799 {
30800 let mut domains_by_study = self
30801 .state
30802 .domains_by_study
30803 .lock()
30804 .expect("asset workflow study-domain lock should not be poisoned");
30805 if !domains_by_study.iter().any(|(existing_study_id, domain)| {
30806 *existing_study_id == study_id && domain.domain_id == domain_id
30807 }) {
30808 domains_by_study.push((study_id, domain));
30809 }
30810 }
30811 Ok(StudyDomainLinkRecord {
30812 study_id,
30813 domain_id,
30814 })
30815 }
30816
30817 async fn list_domains_for_study(
30818 &self,
30819 study_id: StudyId,
30820 ) -> Result<Vec<DomainRecord>, ahri_tre_core::CoreError> {
30821 Ok(self
30822 .state
30823 .domains_by_study
30824 .lock()
30825 .expect("asset workflow study-domain lock should not be poisoned")
30826 .iter()
30827 .filter(|(candidate_study_id, _)| *candidate_study_id == study_id)
30828 .map(|(_, domain)| domain.clone())
30829 .collect())
30830 }
30831 }
30832
30833 struct StatefulAssetStudyRepository {
30834 state: Arc<AssetWorkflowState>,
30835 }
30836
30837 #[async_trait]
30838 impl StudyRepository for StatefulAssetStudyRepository {
30839 async fn list_studies(&self) -> Result<Vec<StudyRecord>, ahri_tre_core::CoreError> {
30840 Ok(self
30841 .state
30842 .studies_by_id
30843 .lock()
30844 .expect("asset workflow study lock should not be poisoned")
30845 .values()
30846 .cloned()
30847 .collect())
30848 }
30849
30850 async fn get_study_by_id(
30851 &self,
30852 study_id: StudyId,
30853 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
30854 Ok(self
30855 .state
30856 .studies_by_id
30857 .lock()
30858 .expect("asset workflow study lock should not be poisoned")
30859 .get(&study_id.0)
30860 .cloned())
30861 }
30862
30863 async fn get_study_by_name(
30864 &self,
30865 name: &str,
30866 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
30867 Ok(self
30868 .state
30869 .studies_by_id
30870 .lock()
30871 .expect("asset workflow study lock should not be poisoned")
30872 .values()
30873 .find(|study| study.name.as_str() == name)
30874 .cloned())
30875 }
30876
30877 async fn search_studies(
30878 &self,
30879 query: &ahri_tre_core::StudySearchQuery,
30880 ) -> Result<
30881 ahri_tre_core::SearchPage<StudyRecord, ahri_tre_core::StudySearchCursorAnchor>,
30882 ahri_tre_core::CoreError,
30883 > {
30884 let studies = self
30885 .state
30886 .studies_by_id
30887 .lock()
30888 .expect("asset workflow study lock should not be poisoned")
30889 .values()
30890 .cloned()
30891 .collect::<Vec<_>>();
30892 let domains_by_study = self
30893 .state
30894 .domains_by_study
30895 .lock()
30896 .expect("asset workflow study-domain lock should not be poisoned")
30897 .clone();
30898 let tags = self
30899 .state
30900 .tags
30901 .lock()
30902 .expect("tag lock should not be poisoned")
30903 .clone();
30904 let tag_links = self
30905 .state
30906 .tag_links
30907 .lock()
30908 .expect("tag link lock should not be poisoned")
30909 .clone();
30910 let mut matched = studies
30911 .into_iter()
30912 .filter(|study| {
30913 let domains = domains_by_study
30914 .iter()
30915 .filter(|(study_id, _)| *study_id == study.study_id)
30916 .map(|(_, domain)| domain.clone())
30917 .collect::<Vec<_>>();
30918 let study_tags = tag_links
30919 .iter()
30920 .filter_map(|(target, tag_id)| match target {
30921 TagAttachmentTarget::Study(study_id) if *study_id == study.study_id => {
30922 tags.iter().find(|tag| tag.tag_id == *tag_id).cloned()
30923 }
30924 _ => None,
30925 })
30926 .collect::<Vec<_>>();
30927 test_study_matches_search_query(study, &domains, &study_tags, query)
30928 })
30929 .collect::<Vec<_>>();
30930
30931 matched.sort_by(test_study_search_sort_key);
30932 if let Some(after) = query.after.as_ref() {
30933 matched
30934 .retain(|study| test_study_sort_tuple(study) > test_study_cursor_tuple(after));
30935 }
30936
30937 let limit = usize::from(query.limit);
30938 let has_more = matched.len() > limit;
30939 if has_more {
30940 matched.truncate(limit);
30941 }
30942 Ok(ahri_tre_core::SearchPage {
30943 items: matched.clone(),
30944 next_cursor: if has_more {
30945 matched
30946 .last()
30947 .map(|study| ahri_tre_core::StudySearchCursorAnchor {
30948 name: study.name.as_str().to_string(),
30949 study_id: study.study_id,
30950 })
30951 } else {
30952 None
30953 },
30954 })
30955 }
30956
30957 async fn create_study(
30958 &self,
30959 study: &NewStudyRecord,
30960 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
30961 if let Some(message) = self
30962 .state
30963 .fail_next_create_study
30964 .lock()
30965 .expect("create study failure lock should not be poisoned")
30966 .take()
30967 {
30968 return Err(ahri_tre_core::CoreError::Infrastructure(message));
30969 }
30970 let study = StudyRecord {
30971 study_id: StudyId(uuid::Uuid::new_v4()),
30972 name: study.name.clone(),
30973 description: study.description.clone(),
30974 documentation: study.documentation.clone(),
30975 agent_instructions: study.agent_instructions.clone(),
30976 external_id: study.external_id.clone(),
30977 study_type_id: study.study_type_id,
30978 date_created: study.date_created,
30979 created_by: study.created_by.clone(),
30980 };
30981 self.save_study(&study).await
30982 }
30983
30984 async fn save_study(
30985 &self,
30986 study: &StudyRecord,
30987 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
30988 let mut study = study.clone();
30989 if study.created_by.is_none() {
30990 study.created_by = Some("test_current_user".to_string());
30991 }
30992 self.state
30993 .studies_by_id
30994 .lock()
30995 .expect("asset workflow study lock should not be poisoned")
30996 .insert(study.study_id.0, study.clone());
30997 Ok(study)
30998 }
30999
31000 async fn delete_study_record(
31001 &self,
31002 study_id: StudyId,
31003 ) -> Result<bool, ahri_tre_core::CoreError> {
31004 let removed = self
31005 .state
31006 .studies_by_id
31007 .lock()
31008 .expect("asset workflow study lock should not be poisoned")
31009 .remove(&study_id.0)
31010 .is_some();
31011 self.state
31012 .domains_by_study
31013 .lock()
31014 .expect("study-domain lock should not be poisoned")
31015 .retain(|(linked_study_id, _)| *linked_study_id != study_id);
31016 self.state
31017 .access_grants
31018 .lock()
31019 .expect("study access lock should not be poisoned")
31020 .retain(|grant| grant.study_id != study_id);
31021 self.state
31022 .custodians
31023 .lock()
31024 .expect("study custodian lock should not be poisoned")
31025 .retain(|custodian| custodian.study_id != study_id);
31026 Ok(removed)
31027 }
31028 }
31029
31030 fn test_study_matches_search_query(
31031 study: &StudyRecord,
31032 domains: &[DomainRecord],
31033 tags: &[ahri_tre_types::TagRecord],
31034 query: &ahri_tre_core::StudySearchQuery,
31035 ) -> bool {
31036 let matches = |predicate: &ahri_tre_core::StudySearchPredicate| -> bool {
31037 match predicate {
31038 ahri_tre_core::StudySearchPredicate::Text { field, mode, value } => {
31039 let candidate = match field {
31040 ahri_tre_core::StudySearchTextField::Name => Some(study.name.as_str()),
31041 ahri_tre_core::StudySearchTextField::Title => {
31042 study.documentation.as_deref()
31043 }
31044 ahri_tre_core::StudySearchTextField::Description => {
31045 study.description.as_deref()
31046 }
31047 };
31048 candidate.is_some_and(|candidate| {
31049 ahri_tre_core::search_text_matches(candidate, *mode, value)
31050 })
31051 }
31052 ahri_tre_core::StudySearchPredicate::Domain(selector) => {
31053 domains.iter().any(|domain| match selector {
31054 ahri_tre_core::StudySearchDomainSelector::Id(domain_id) => {
31055 domain.domain_id == *domain_id
31056 }
31057 ahri_tre_core::StudySearchDomainSelector::Name(name) => {
31058 domain.name.as_str() == name
31059 }
31060 })
31061 }
31062 ahri_tre_core::StudySearchPredicate::Tag(selector) => {
31063 ahri_tre_core::search_tags_match_selector(tags, selector)
31064 }
31065 }
31066 };
31067
31068 ahri_tre_core::predicate_set_matches(query.kind, &query.predicates, matches)
31069 }
31070
31071 fn test_study_search_sort_key(left: &StudyRecord, right: &StudyRecord) -> std::cmp::Ordering {
31072 test_study_sort_tuple(left).cmp(&test_study_sort_tuple(right))
31073 }
31074
31075 fn test_study_sort_tuple(study: &StudyRecord) -> (String, String, uuid::Uuid) {
31076 (
31077 study.name.as_str().to_ascii_lowercase(),
31078 study.name.as_str().to_string(),
31079 study.study_id.0,
31080 )
31081 }
31082
31083 fn test_study_cursor_tuple(
31084 anchor: &ahri_tre_core::StudySearchCursorAnchor,
31085 ) -> (String, String, uuid::Uuid) {
31086 (
31087 anchor.name.to_ascii_lowercase(),
31088 anchor.name.clone(),
31089 anchor.study_id.0,
31090 )
31091 }
31092
31093 fn test_latest_searchable_dataset_version(
31094 versions: &[AssetVersionRecord],
31095 dataset_ids: &[uuid::Uuid],
31096 withdrawn_ids: &[uuid::Uuid],
31097 ) -> Option<AssetVersionRecord> {
31098 versions
31099 .iter()
31100 .filter(|version| {
31101 dataset_ids.contains(&version.version_id.0)
31102 && version.is_latest.is_some()
31103 && !withdrawn_ids.contains(&version.version_id.0)
31104 })
31105 .max_by_key(|version| (version.major, version.minor, version.patch))
31106 .cloned()
31107 .map(|mut version| {
31108 version.is_latest = Some(true);
31109 version
31110 })
31111 }
31112
31113 fn test_dataset_matches_search_query(
31114 study: &StudyRecord,
31115 asset: &AssetRecord,
31116 latest_version: &AssetVersionRecord,
31117 domains: &[DomainRecord],
31118 tags: &[ahri_tre_types::TagRecord],
31119 query: &ahri_tre_core::DatasetSearchQuery,
31120 ) -> bool {
31121 let matches = |predicate: &ahri_tre_core::DatasetSearchPredicate| -> bool {
31122 match predicate {
31123 ahri_tre_core::DatasetSearchPredicate::Text { field, mode, value } => {
31124 let candidate = match field {
31125 ahri_tre_core::DatasetSearchTextField::Name => Some(asset.name.as_str()),
31126 ahri_tre_core::DatasetSearchTextField::Description => {
31127 asset.description.as_deref()
31128 }
31129 ahri_tre_core::DatasetSearchTextField::VersionLabel => {
31130 latest_version.version_label.as_deref()
31131 }
31132 };
31133 candidate.is_some_and(|candidate| {
31134 ahri_tre_core::search_text_matches(candidate, *mode, value)
31135 })
31136 }
31137 ahri_tre_core::DatasetSearchPredicate::Study(selector) => match selector {
31138 ahri_tre_core::DatasetSearchStudySelector::Id(study_id) => {
31139 study.study_id == *study_id
31140 }
31141 ahri_tre_core::DatasetSearchStudySelector::Name { name, domain } => {
31142 study.name.as_str() == name
31143 && domain.as_ref().is_none_or(|domain_selector| {
31144 domains.iter().any(|domain| match domain_selector {
31145 ahri_tre_core::DatasetSearchDomainSelector::Id(domain_id) => {
31146 domain.domain_id == *domain_id
31147 }
31148 ahri_tre_core::DatasetSearchDomainSelector::Name(
31149 domain_name,
31150 ) => domain.name.as_str() == domain_name,
31151 })
31152 })
31153 }
31154 },
31155 ahri_tre_core::DatasetSearchPredicate::Domain(selector) => {
31156 domains.iter().any(|domain| match selector {
31157 ahri_tre_core::DatasetSearchDomainSelector::Id(domain_id) => {
31158 domain.domain_id == *domain_id
31159 }
31160 ahri_tre_core::DatasetSearchDomainSelector::Name(name) => {
31161 domain.name.as_str() == name
31162 }
31163 })
31164 }
31165 ahri_tre_core::DatasetSearchPredicate::VersionLabel(label) => latest_version
31166 .version_label
31167 .as_deref()
31168 .is_some_and(|version_label| version_label == label),
31169 ahri_tre_core::DatasetSearchPredicate::Tag(selector) => {
31170 ahri_tre_core::search_tags_match_selector(tags, selector)
31171 }
31172 }
31173 };
31174
31175 ahri_tre_core::predicate_set_matches(query.kind, &query.predicates, matches)
31176 }
31177
31178 fn test_dataset_search_sort_key(
31179 left: &ahri_tre_core::DatasetSearchRecord,
31180 right: &ahri_tre_core::DatasetSearchRecord,
31181 ) -> std::cmp::Ordering {
31182 test_dataset_sort_tuple(left).cmp(&test_dataset_sort_tuple(right))
31183 }
31184
31185 fn test_dataset_sort_tuple(
31186 dataset: &ahri_tre_core::DatasetSearchRecord,
31187 ) -> (String, String, uuid::Uuid, uuid::Uuid) {
31188 (
31189 dataset.asset.name.as_str().to_ascii_lowercase(),
31190 dataset.asset.name.as_str().to_string(),
31191 dataset.study_id.0,
31192 dataset.asset.asset_id.0,
31193 )
31194 }
31195
31196 fn test_dataset_cursor_tuple(
31197 anchor: &ahri_tre_core::DatasetSearchCursorAnchor,
31198 ) -> (String, String, uuid::Uuid, uuid::Uuid) {
31199 (
31200 anchor.name.to_ascii_lowercase(),
31201 anchor.name.clone(),
31202 anchor.study_id.0,
31203 anchor.asset_id.0,
31204 )
31205 }
31206
31207 struct StatefulAssetRepository {
31208 state: Arc<AssetWorkflowState>,
31209 }
31210
31211 #[async_trait]
31212 impl AssetRepository for StatefulAssetRepository {
31213 async fn asset_classification(
31214 &self,
31215 _asset: AssetId,
31216 ) -> Result<Option<ahri_tre_types::AssetClassification>, ahri_tre_core::CoreError> {
31217 Ok(Some(ahri_tre_types::AssetClassification {
31218 risk: ahri_tre_types::AssetRisk::High,
31219 revision: 1,
31220 evidence_id: uuid::Uuid::nil(),
31221 evidence_durable: true,
31222 }))
31223 }
31224 async fn validate_dataset_derivation(
31225 &self,
31226 _inputs: &[ahri_tre_types::VersionId],
31227 _risk: ahri_tre_types::AssetRisk,
31228 _inherit: bool,
31229 ) -> Result<(), ahri_tre_core::CoreError> {
31230 Ok(())
31231 }
31232
31233 async fn list_assets_for_study(
31234 &self,
31235 study_id: StudyId,
31236 ) -> Result<Vec<AssetRecord>, ahri_tre_core::CoreError> {
31237 let mut assets: Vec<_> = self
31238 .state
31239 .assets_by_id
31240 .lock()
31241 .expect("asset lock should not be poisoned")
31242 .values()
31243 .filter(|asset| asset.study_id == study_id)
31244 .cloned()
31245 .collect();
31246 assets.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str()));
31247 Ok(assets)
31248 }
31249
31250 async fn get_asset_by_id(
31251 &self,
31252 asset_id: AssetId,
31253 ) -> Result<Option<AssetRecord>, ahri_tre_core::CoreError> {
31254 Ok(self
31255 .state
31256 .assets_by_id
31257 .lock()
31258 .expect("asset lock should not be poisoned")
31259 .get(&asset_id.0)
31260 .cloned())
31261 }
31262
31263 async fn get_asset_by_name(
31264 &self,
31265 study_id: StudyId,
31266 name: &str,
31267 ) -> Result<Option<AssetRecord>, ahri_tre_core::CoreError> {
31268 Ok(self
31269 .state
31270 .assets_by_id
31271 .lock()
31272 .expect("asset lock should not be poisoned")
31273 .values()
31274 .find(|asset| asset.study_id == study_id && asset.name.as_str() == name)
31275 .cloned())
31276 }
31277
31278 async fn create_asset(
31279 &self,
31280 asset: &ahri_tre_types::NewAssetRecord,
31281 ) -> Result<AssetRecord, ahri_tre_core::CoreError> {
31282 if self
31283 .get_asset_by_name(asset.study_id, asset.name.as_str())
31284 .await?
31285 .is_some()
31286 {
31287 return Err(ahri_tre_core::CoreError::Conflict(
31288 "Asset name already registered in Study".into(),
31289 ));
31290 }
31291 let asset = AssetRecord {
31292 asset_id: AssetId(uuid::Uuid::new_v4()),
31293 study_id: asset.study_id,
31294 name: asset.name.clone(),
31295 description: asset.description.clone(),
31296 agent_instructions: asset.agent_instructions.clone(),
31297 asset_type: asset.asset_type,
31298 date_created: asset.date_created,
31299 created_by: asset.created_by.clone(),
31300 };
31301 self.save_asset(&asset).await
31302 }
31303
31304 async fn save_asset(
31305 &self,
31306 asset: &AssetRecord,
31307 ) -> Result<AssetRecord, ahri_tre_core::CoreError> {
31308 if let Some(message) = self
31309 .state
31310 .fail_next_save_asset
31311 .lock()
31312 .expect("save asset failure lock should not be poisoned")
31313 .take()
31314 {
31315 return Err(ahri_tre_core::CoreError::Infrastructure(message));
31316 }
31317 self.state
31318 .assets_by_id
31319 .lock()
31320 .expect("asset lock should not be poisoned")
31321 .insert(asset.asset_id.0, asset.clone());
31322 Ok(asset.clone())
31323 }
31324
31325 async fn list_asset_versions(
31326 &self,
31327 asset_id: AssetId,
31328 ) -> Result<Vec<AssetVersionRecord>, ahri_tre_core::CoreError> {
31329 let mut versions = self
31330 .state
31331 .versions_by_asset
31332 .lock()
31333 .expect("asset version lock should not be poisoned")
31334 .get(&asset_id.0)
31335 .cloned()
31336 .unwrap_or_default();
31337 versions.sort_by_key(|version| (version.major, version.minor, version.patch));
31338 Ok(versions)
31339 }
31340
31341 async fn get_asset_version(
31342 &self,
31343 version_id: VersionId,
31344 ) -> Result<Option<AssetVersionRecord>, ahri_tre_core::CoreError> {
31345 Ok(self
31346 .state
31347 .versions_by_asset
31348 .lock()
31349 .expect("asset version lock should not be poisoned")
31350 .values()
31351 .flat_map(|versions| versions.iter())
31352 .find(|version| version.version_id == version_id)
31353 .cloned())
31354 }
31355
31356 async fn create_asset_version(
31357 &self,
31358 version: &ahri_tre_types::NewAssetVersionRecord,
31359 ) -> Result<AssetVersionRecord, ahri_tre_core::CoreError> {
31360 let version = AssetVersionRecord {
31361 version_id: VersionId(uuid::Uuid::new_v4()),
31362 asset_id: version.asset_id,
31363 major: version.major,
31364 minor: version.minor,
31365 patch: version.patch,
31366 version_label: version.version_label.clone(),
31367 version_note: version.version_note.clone(),
31368 is_latest: version.is_latest,
31369 doi: version.doi.clone(),
31370 created_at: version.created_at,
31371 created_by: version.created_by.clone(),
31372 };
31373 self.save_asset_version(&version).await
31374 }
31375
31376 async fn save_asset_version(
31377 &self,
31378 version: &AssetVersionRecord,
31379 ) -> Result<AssetVersionRecord, ahri_tre_core::CoreError> {
31380 if let Some(message) = self
31381 .state
31382 .fail_next_save_asset_version
31383 .lock()
31384 .expect("save asset version failure lock should not be poisoned")
31385 .take()
31386 {
31387 return Err(ahri_tre_core::CoreError::Infrastructure(message));
31388 }
31389 let mut version = version.clone();
31390 version.version_label = Some(ahri_tre_core::asset_version_label(
31391 version.major,
31392 version.minor,
31393 version.patch,
31394 ));
31395 version.is_latest = Some(version.is_latest.unwrap_or(true));
31396
31397 let mut versions_by_asset = self
31398 .state
31399 .versions_by_asset
31400 .lock()
31401 .expect("asset version lock should not be poisoned");
31402 let versions = versions_by_asset.entry(version.asset_id.0).or_default();
31403 if version.is_latest == Some(true) {
31404 for existing in versions.iter_mut() {
31405 existing.is_latest = Some(false);
31406 }
31407 }
31408 if let Some(existing) = versions
31409 .iter_mut()
31410 .find(|existing| existing.version_id == version.version_id)
31411 {
31412 *existing = version.clone();
31413 } else {
31414 versions.push(version.clone());
31415 }
31416 Ok(version)
31417 }
31418
31419 async fn mark_asset_version_latest(
31420 &self,
31421 asset_id: AssetId,
31422 version_id: VersionId,
31423 ) -> Result<AssetVersionRecord, ahri_tre_core::CoreError> {
31424 let mut versions_by_asset = self
31425 .state
31426 .versions_by_asset
31427 .lock()
31428 .expect("asset version lock should not be poisoned");
31429 let versions = versions_by_asset.entry(asset_id.0).or_default();
31430 if !versions
31431 .iter()
31432 .any(|version| version.version_id == version_id)
31433 {
31434 return Err(ahri_tre_core::CoreError::NotFound(format!(
31435 "asset version not found: {}",
31436 version_id.0
31437 )));
31438 }
31439 let mut marked = None;
31440 for version in versions.iter_mut() {
31441 version.is_latest = Some(version.version_id == version_id);
31442 if version.version_id == version_id {
31443 marked = Some(version.clone());
31444 }
31445 }
31446 marked.ok_or_else(|| {
31447 ahri_tre_core::CoreError::NotFound(format!(
31448 "asset version not found: {}",
31449 version_id.0
31450 ))
31451 })
31452 }
31453
31454 async fn list_datasets_for_asset(
31455 &self,
31456 asset_id: AssetId,
31457 ) -> Result<Vec<DatasetRecord>, ahri_tre_core::CoreError> {
31458 let versions = self
31459 .list_asset_versions(asset_id)
31460 .await?
31461 .into_iter()
31462 .map(|version| version.version_id.0)
31463 .collect::<Vec<_>>();
31464 let datasets = self
31465 .state
31466 .datasets_by_version
31467 .lock()
31468 .expect("dataset lock should not be poisoned");
31469 Ok(versions
31470 .iter()
31471 .filter_map(|version_id| datasets.get(version_id).cloned())
31472 .collect())
31473 }
31474
31475 async fn search_datasets(
31476 &self,
31477 query: &ahri_tre_core::DatasetSearchQuery,
31478 ) -> Result<
31479 ahri_tre_core::SearchPage<
31480 ahri_tre_core::DatasetSearchRecord,
31481 ahri_tre_core::DatasetSearchCursorAnchor,
31482 >,
31483 ahri_tre_core::CoreError,
31484 > {
31485 let studies = self
31486 .state
31487 .studies_by_id
31488 .lock()
31489 .expect("asset workflow study lock should not be poisoned")
31490 .values()
31491 .cloned()
31492 .collect::<Vec<_>>();
31493 let domains_by_study = self
31494 .state
31495 .domains_by_study
31496 .lock()
31497 .expect("asset workflow study-domain lock should not be poisoned")
31498 .clone();
31499 let tags = self
31500 .state
31501 .tags
31502 .lock()
31503 .expect("tag lock should not be poisoned")
31504 .clone();
31505 let tag_links = self
31506 .state
31507 .tag_links
31508 .lock()
31509 .expect("tag link lock should not be poisoned")
31510 .clone();
31511 let assets = self
31512 .state
31513 .assets_by_id
31514 .lock()
31515 .expect("asset lock should not be poisoned")
31516 .values()
31517 .cloned()
31518 .collect::<Vec<_>>();
31519 let versions_by_asset = self
31520 .state
31521 .versions_by_asset
31522 .lock()
31523 .expect("asset version lock should not be poisoned")
31524 .clone();
31525 let datasets_by_version = self
31526 .state
31527 .datasets_by_version
31528 .lock()
31529 .expect("dataset lock should not be poisoned")
31530 .clone();
31531 let withdrawals_by_version = self
31532 .state
31533 .dataset_withdrawals_by_version
31534 .lock()
31535 .expect("dataset withdrawal lock should not be poisoned")
31536 .clone();
31537
31538 let mut matched = studies
31539 .into_iter()
31540 .flat_map(|study| {
31541 let study_domains = domains_by_study
31542 .iter()
31543 .filter(|(study_id, _)| *study_id == study.study_id)
31544 .map(|(_, domain)| domain.clone())
31545 .collect::<Vec<_>>();
31546 assets
31547 .iter()
31548 .filter(move |asset| {
31549 asset.study_id == study.study_id
31550 && asset.asset_type == AssetType::Dataset
31551 })
31552 .filter_map(|asset| {
31553 let versions = versions_by_asset
31554 .get(&asset.asset_id.0)
31555 .cloned()
31556 .unwrap_or_default();
31557 let dataset_ids = versions
31558 .iter()
31559 .filter_map(|version| {
31560 datasets_by_version
31561 .contains_key(&version.version_id.0)
31562 .then_some(version.version_id.0)
31563 })
31564 .collect::<Vec<_>>();
31565 let withdrawn_ids = withdrawals_by_version
31566 .values()
31567 .filter(|withdrawal| withdrawal.asset_id == asset.asset_id)
31568 .map(|withdrawal| withdrawal.dataset_id.0)
31569 .collect::<Vec<_>>();
31570 let latest_version = test_latest_searchable_dataset_version(
31571 &versions,
31572 &dataset_ids,
31573 &withdrawn_ids,
31574 )?;
31575 let dataset_tags = tag_links
31576 .iter()
31577 .filter_map(|(target, tag_id)| match target {
31578 TagAttachmentTarget::AssetVersion(version_id)
31579 if *version_id == latest_version.version_id =>
31580 {
31581 tags.iter().find(|tag| tag.tag_id == *tag_id).cloned()
31582 }
31583 _ => None,
31584 })
31585 .collect::<Vec<_>>();
31586 test_dataset_matches_search_query(
31587 &study,
31588 asset,
31589 &latest_version,
31590 &study_domains,
31591 &dataset_tags,
31592 query,
31593 )
31594 .then_some(
31595 ahri_tre_core::DatasetSearchRecord {
31596 study_id: study.study_id,
31597 asset: asset.clone(),
31598 latest_version,
31599 },
31600 )
31601 })
31602 .collect::<Vec<_>>()
31603 })
31604 .collect::<Vec<_>>();
31605
31606 matched.sort_by(test_dataset_search_sort_key);
31607 if let Some(after) = query.after.as_ref() {
31608 matched.retain(|dataset| {
31609 test_dataset_sort_tuple(dataset) > test_dataset_cursor_tuple(after)
31610 });
31611 }
31612
31613 let limit = usize::from(query.limit);
31614 let has_more = matched.len() > limit;
31615 if has_more {
31616 matched.truncate(limit);
31617 }
31618 Ok(ahri_tre_core::SearchPage {
31619 items: matched.clone(),
31620 next_cursor: if has_more {
31621 matched
31622 .last()
31623 .map(|dataset| ahri_tre_core::DatasetSearchCursorAnchor {
31624 name: dataset.asset.name.as_str().to_string(),
31625 study_id: dataset.study_id,
31626 asset_id: dataset.asset.asset_id,
31627 })
31628 } else {
31629 None
31630 },
31631 })
31632 }
31633
31634 async fn get_dataset(
31635 &self,
31636 dataset_id: VersionId,
31637 ) -> Result<Option<DatasetRecord>, ahri_tre_core::CoreError> {
31638 Ok(self
31639 .state
31640 .datasets_by_version
31641 .lock()
31642 .expect("dataset lock should not be poisoned")
31643 .get(&dataset_id.0)
31644 .cloned())
31645 }
31646
31647 async fn save_dataset(
31648 &self,
31649 dataset: &DatasetRecord,
31650 ) -> Result<DatasetRecord, ahri_tre_core::CoreError> {
31651 if let Some(message) = self
31652 .state
31653 .fail_next_save_dataset
31654 .lock()
31655 .expect("save dataset failure lock should not be poisoned")
31656 .take()
31657 {
31658 return Err(ahri_tre_core::CoreError::Infrastructure(message));
31659 }
31660 self.state
31661 .datasets_by_version
31662 .lock()
31663 .expect("dataset lock should not be poisoned")
31664 .insert(dataset.dataset_id.0, dataset.clone());
31665 Ok(dataset.clone())
31666 }
31667
31668 async fn get_dataset_version_withdrawal(
31669 &self,
31670 dataset_id: VersionId,
31671 ) -> Result<Option<ahri_tre_types::DatasetVersionWithdrawalRecord>, ahri_tre_core::CoreError>
31672 {
31673 Ok(self
31674 .state
31675 .dataset_withdrawals_by_version
31676 .lock()
31677 .expect("dataset withdrawal lock should not be poisoned")
31678 .get(&dataset_id.0)
31679 .cloned())
31680 }
31681
31682 async fn list_dataset_version_withdrawals(
31683 &self,
31684 asset_id: AssetId,
31685 ) -> Result<Vec<ahri_tre_types::DatasetVersionWithdrawalRecord>, ahri_tre_core::CoreError>
31686 {
31687 Ok(self
31688 .state
31689 .dataset_withdrawals_by_version
31690 .lock()
31691 .expect("dataset withdrawal lock should not be poisoned")
31692 .values()
31693 .filter(|withdrawal| withdrawal.asset_id == asset_id)
31694 .cloned()
31695 .collect())
31696 }
31697
31698 async fn save_dataset_version_withdrawal(
31699 &self,
31700 withdrawal: &ahri_tre_types::NewDatasetVersionWithdrawalRecord,
31701 ) -> Result<ahri_tre_types::DatasetVersionWithdrawalRecord, ahri_tre_core::CoreError>
31702 {
31703 let mut withdrawals = self
31704 .state
31705 .dataset_withdrawals_by_version
31706 .lock()
31707 .expect("dataset withdrawal lock should not be poisoned");
31708 let record = withdrawals
31709 .entry(withdrawal.dataset_id.0)
31710 .or_insert_with(|| ahri_tre_types::DatasetVersionWithdrawalRecord {
31711 dataset_id: withdrawal.dataset_id,
31712 asset_id: withdrawal.asset_id,
31713 withdrawn_at: Some(Utc::now()),
31714 withdrawn_by: withdrawal.withdrawn_by.clone(),
31715 reason: withdrawal.reason.clone(),
31716 tombstone_state: withdrawal.tombstone_state.clone(),
31717 provenance: withdrawal.provenance.clone(),
31718 drop_lake_table: withdrawal.drop_lake_table,
31719 })
31720 .clone();
31721 Ok(record)
31722 }
31723
31724 async fn deactivate_dataset_version_links(
31725 &self,
31726 dataset_id: VersionId,
31727 ) -> Result<(), ahri_tre_core::CoreError> {
31728 self.state
31729 .dataset_variables
31730 .lock()
31731 .expect("dataset variable lock should not be poisoned")
31732 .retain(|link| link.dataset_id != dataset_id);
31733 Ok(())
31734 }
31735
31736 async fn clear_asset_version_latest(
31737 &self,
31738 asset_id: AssetId,
31739 version_id: VersionId,
31740 ) -> Result<AssetVersionRecord, ahri_tre_core::CoreError> {
31741 let mut versions_by_asset = self
31742 .state
31743 .versions_by_asset
31744 .lock()
31745 .expect("asset version lock should not be poisoned");
31746 let versions = versions_by_asset.entry(asset_id.0).or_default();
31747 let mut cleared = None;
31748 for version in versions.iter_mut() {
31749 if version.version_id == version_id {
31750 version.is_latest = Some(false);
31751 cleared = Some(version.clone());
31752 }
31753 }
31754 cleared.ok_or_else(|| {
31755 ahri_tre_core::CoreError::NotFound(format!(
31756 "asset version not found: {}",
31757 version_id.0
31758 ))
31759 })
31760 }
31761
31762 async fn list_datafiles_for_asset(
31763 &self,
31764 asset_id: AssetId,
31765 ) -> Result<Vec<DataFileRecord>, ahri_tre_core::CoreError> {
31766 let versions = self
31767 .list_asset_versions(asset_id)
31768 .await?
31769 .into_iter()
31770 .map(|version| version.version_id.0)
31771 .collect::<Vec<_>>();
31772 let datafiles = self
31773 .state
31774 .datafiles_by_version
31775 .lock()
31776 .expect("datafile lock should not be poisoned");
31777 Ok(versions
31778 .iter()
31779 .filter_map(|version_id| datafiles.get(version_id).cloned())
31780 .collect())
31781 }
31782
31783 async fn get_datafile(
31784 &self,
31785 datafile_id: VersionId,
31786 ) -> Result<Option<DataFileRecord>, ahri_tre_core::CoreError> {
31787 Ok(self
31788 .state
31789 .datafiles_by_version
31790 .lock()
31791 .expect("datafile lock should not be poisoned")
31792 .get(&datafile_id.0)
31793 .cloned())
31794 }
31795
31796 async fn save_datafile(
31797 &self,
31798 datafile: &DataFileRecord,
31799 ) -> Result<DataFileRecord, ahri_tre_core::CoreError> {
31800 if let Some(message) = self
31801 .state
31802 .fail_next_save_datafile
31803 .lock()
31804 .expect("save datafile failure lock should not be poisoned")
31805 .take()
31806 {
31807 return Err(ahri_tre_core::CoreError::Infrastructure(message));
31808 }
31809 ahri_tre_core::validate_datafile_record(datafile)?;
31810 self.state
31811 .datafiles_by_version
31812 .lock()
31813 .expect("datafile lock should not be poisoned")
31814 .insert(datafile.datafile_id.0, datafile.clone());
31815 Ok(datafile.clone())
31816 }
31817
31818 async fn delete_datafile_record(
31819 &self,
31820 datafile_id: VersionId,
31821 ) -> Result<bool, ahri_tre_core::CoreError> {
31822 Ok(self
31823 .state
31824 .datafiles_by_version
31825 .lock()
31826 .expect("datafile lock should not be poisoned")
31827 .remove(&datafile_id.0)
31828 .is_some())
31829 }
31830
31831 async fn delete_datafile_version_metadata(
31832 &self,
31833 version_id: VersionId,
31834 latest_promotion: Option<(AssetId, Option<VersionId>)>,
31835 ) -> Result<bool, ahri_tre_core::CoreError> {
31836 if let Some((asset_id, promoted)) = latest_promotion {
31837 self.clear_asset_version_latest(asset_id, version_id)
31838 .await?;
31839 if let Some(promoted) = promoted {
31840 self.mark_asset_version_latest(asset_id, promoted).await?;
31841 }
31842 }
31843 self.delete_asset_version_record(version_id).await
31844 }
31845
31846 async fn delete_asset_version_record(
31847 &self,
31848 version_id: VersionId,
31849 ) -> Result<bool, ahri_tre_core::CoreError> {
31850 self.state
31851 .datasets_by_version
31852 .lock()
31853 .expect("dataset lock should not be poisoned")
31854 .remove(&version_id.0);
31855 self.state
31856 .datafiles_by_version
31857 .lock()
31858 .expect("datafile lock should not be poisoned")
31859 .remove(&version_id.0);
31860 self.state
31861 .dataset_variables
31862 .lock()
31863 .expect("dataset variable lock should not be poisoned")
31864 .retain(|link| link.dataset_id != version_id);
31865 let mut removed = false;
31866 let mut versions_by_asset = self
31867 .state
31868 .versions_by_asset
31869 .lock()
31870 .expect("asset version lock should not be poisoned");
31871 for versions in versions_by_asset.values_mut() {
31872 let before = versions.len();
31873 versions.retain(|version| version.version_id != version_id);
31874 removed |= versions.len() != before;
31875 }
31876 Ok(removed)
31877 }
31878
31879 async fn delete_asset_record(
31880 &self,
31881 asset_id: AssetId,
31882 ) -> Result<bool, ahri_tre_core::CoreError> {
31883 self.state
31884 .versions_by_asset
31885 .lock()
31886 .expect("asset version lock should not be poisoned")
31887 .remove(&asset_id.0);
31888 Ok(self
31889 .state
31890 .assets_by_id
31891 .lock()
31892 .expect("asset lock should not be poisoned")
31893 .remove(&asset_id.0)
31894 .is_some())
31895 }
31896
31897 async fn list_asset_version_duo_restrictions(
31898 &self,
31899 version_id: VersionId,
31900 ) -> Result<Vec<AssetVersionDuoRestrictionRecord>, ahri_tre_core::CoreError> {
31901 Ok(self
31902 .state
31903 .asset_version_duo_restrictions
31904 .lock()
31905 .expect("asset version DUO restriction lock should not be poisoned")
31906 .iter()
31907 .filter(|restriction| restriction.version_id == version_id)
31908 .cloned()
31909 .collect())
31910 }
31911
31912 async fn create_asset_version_duo_restriction(
31913 &self,
31914 restriction: &NewAssetVersionDuoRestrictionRecord,
31915 ) -> Result<AssetVersionDuoRestrictionRecord, ahri_tre_core::CoreError> {
31916 let mut restrictions = self
31917 .state
31918 .asset_version_duo_restrictions
31919 .lock()
31920 .expect("asset version DUO restriction lock should not be poisoned");
31921 let restriction_id = restrictions
31922 .iter()
31923 .map(|restriction| restriction.asset_version_duo_restriction_id.0)
31924 .max()
31925 .unwrap_or(0)
31926 + 1;
31927 let persisted = AssetVersionDuoRestrictionRecord {
31928 asset_version_duo_restriction_id: AssetVersionDuoRestrictionId(restriction_id),
31929 version_id: restriction.version_id,
31930 duo_id: restriction.duo_id.clone(),
31931 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
31932 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
31933 qualifier_text: restriction.qualifier_text.clone(),
31934 qualifier_date: restriction.qualifier_date,
31935 qualifier_integer: restriction.qualifier_integer,
31936 };
31937 restrictions.push(persisted.clone());
31938 Ok(persisted)
31939 }
31940
31941 async fn save_asset_version_duo_restriction(
31942 &self,
31943 restriction: &AssetVersionDuoRestrictionRecord,
31944 ) -> Result<AssetVersionDuoRestrictionRecord, ahri_tre_core::CoreError> {
31945 let mut restrictions = self
31946 .state
31947 .asset_version_duo_restrictions
31948 .lock()
31949 .expect("asset version DUO restriction lock should not be poisoned");
31950 if let Some(existing) = restrictions.iter_mut().find(|existing| {
31951 existing.asset_version_duo_restriction_id
31952 == restriction.asset_version_duo_restriction_id
31953 }) {
31954 *existing = restriction.clone();
31955 } else {
31956 restrictions.push(restriction.clone());
31957 }
31958 Ok(restriction.clone())
31959 }
31960
31961 async fn replace_asset_version_duo_restrictions(
31962 &self,
31963 version_id: VersionId,
31964 restrictions: &[NewAssetVersionDuoRestrictionRecord],
31965 ) -> Result<Vec<AssetVersionDuoRestrictionRecord>, ahri_tre_core::CoreError> {
31966 let mut persisted = self
31967 .state
31968 .asset_version_duo_restrictions
31969 .lock()
31970 .expect("asset version DUO restriction lock should not be poisoned");
31971 persisted.retain(|restriction| restriction.version_id != version_id);
31972 let next_id = persisted
31973 .iter()
31974 .map(|restriction| restriction.asset_version_duo_restriction_id.0)
31975 .max()
31976 .unwrap_or(0)
31977 + 1;
31978 let mut replaced = Vec::with_capacity(restrictions.len());
31979 for (offset, restriction) in restrictions.iter().enumerate() {
31980 let row = AssetVersionDuoRestrictionRecord {
31981 asset_version_duo_restriction_id: AssetVersionDuoRestrictionId(
31982 next_id + offset as i64,
31983 ),
31984 version_id,
31985 duo_id: restriction.duo_id.clone(),
31986 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
31987 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
31988 qualifier_text: restriction.qualifier_text.clone(),
31989 qualifier_date: restriction.qualifier_date,
31990 qualifier_integer: restriction.qualifier_integer,
31991 };
31992 persisted.push(row.clone());
31993 replaced.push(row);
31994 }
31995 Ok(replaced)
31996 }
31997
31998 async fn clear_asset_version_duo_restrictions(
31999 &self,
32000 version_id: VersionId,
32001 ) -> Result<Vec<AssetVersionDuoRestrictionRecord>, ahri_tre_core::CoreError> {
32002 let mut persisted = self
32003 .state
32004 .asset_version_duo_restrictions
32005 .lock()
32006 .expect("asset version DUO restriction lock should not be poisoned");
32007 persisted.retain(|restriction| restriction.version_id != version_id);
32008 Ok(Vec::new())
32009 }
32010 }
32011
32012 struct StatefulVariableRepository {
32013 state: Arc<AssetWorkflowState>,
32014 }
32015
32016 #[async_trait]
32017 impl VariableRepository for StatefulVariableRepository {
32018 async fn list_value_types(
32019 &self,
32020 ) -> Result<Vec<ahri_tre_types::ValueTypeRecord>, ahri_tre_core::CoreError> {
32021 Ok(default_value_type_records())
32022 }
32023
32024 async fn list_domain_variables(
32025 &self,
32026 domain_id: DomainId,
32027 ) -> Result<Vec<VariableRecord>, ahri_tre_core::CoreError> {
32028 if let Some(message) = self
32029 .state
32030 .fail_next_list_domain_variables
32031 .lock()
32032 .expect("list domain variables failure lock should not be poisoned")
32033 .take()
32034 {
32035 return Err(ahri_tre_core::CoreError::Infrastructure(message));
32036 }
32037 let mut variables: Vec<_> = self
32038 .state
32039 .variables_by_id
32040 .lock()
32041 .expect("variable lock should not be poisoned")
32042 .values()
32043 .filter(|variable| variable.domain_id == domain_id)
32044 .cloned()
32045 .collect();
32046 variables.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str()));
32047 Ok(variables)
32048 }
32049
32050 async fn get_variable(
32051 &self,
32052 variable_id: VariableId,
32053 ) -> Result<Option<VariableRecord>, ahri_tre_core::CoreError> {
32054 Ok(self
32055 .state
32056 .variables_by_id
32057 .lock()
32058 .expect("variable lock should not be poisoned")
32059 .get(&variable_id.0)
32060 .cloned())
32061 }
32062
32063 async fn create_variable(
32064 &self,
32065 variable: &ahri_tre_types::NewVariableRecord,
32066 ) -> Result<VariableRecord, ahri_tre_core::CoreError> {
32067 let mut variables = self
32068 .state
32069 .variables_by_id
32070 .lock()
32071 .expect("variable lock should not be poisoned");
32072 let variable_id = VariableId(variables.keys().next_back().copied().unwrap_or(0) + 1);
32073 let record = VariableRecord {
32074 variable_id,
32075 domain_id: variable.domain_id,
32076 name: variable.name.clone(),
32077 value_type_id: variable.value_type_id,
32078 value_format: variable.value_format.clone(),
32079 vocabulary_id: variable.vocabulary_id,
32080 key_role: variable.key_role,
32081 description: variable.description.clone(),
32082 note: variable.note.clone(),
32083 ontology_namespace: variable.ontology_namespace.clone(),
32084 ontology_class: variable.ontology_class.clone(),
32085 };
32086 variables.insert(record.variable_id.0, record.clone());
32087 Ok(record)
32088 }
32089
32090 async fn save_variable(
32091 &self,
32092 variable: &VariableRecord,
32093 ) -> Result<VariableRecord, ahri_tre_core::CoreError> {
32094 self.state
32095 .variables_by_id
32096 .lock()
32097 .expect("variable lock should not be poisoned")
32098 .insert(variable.variable_id.0, variable.clone());
32099 Ok(variable.clone())
32100 }
32101
32102 async fn update_variable_metadata(
32103 &self,
32104 variable: &VariableRecord,
32105 ) -> Result<VariableRecord, ahri_tre_core::CoreError> {
32106 self.save_variable(variable).await
32107 }
32108
32109 async fn delete_variable_record(
32110 &self,
32111 variable_id: VariableId,
32112 ) -> Result<bool, ahri_tre_core::CoreError> {
32113 self.state
32114 .dataset_variables
32115 .lock()
32116 .expect("dataset variable lock should not be poisoned")
32117 .retain(|link| link.variable_id != variable_id);
32118 Ok(self
32119 .state
32120 .variables_by_id
32121 .lock()
32122 .expect("variable lock should not be poisoned")
32123 .remove(&variable_id.0)
32124 .is_some())
32125 }
32126
32127 async fn link_dataset_variable(
32128 &self,
32129 dataset_id: VersionId,
32130 variable_id: VariableId,
32131 row_role: KeyRole,
32132 ) -> Result<ahri_tre_types::DatasetVariableLinkRecord, ahri_tre_core::CoreError> {
32133 if let Some(message) = self
32134 .state
32135 .fail_next_link_dataset_variable
32136 .lock()
32137 .expect("dataset-variable failure lock should not be poisoned")
32138 .take()
32139 {
32140 return Err(ahri_tre_core::CoreError::Infrastructure(message));
32141 }
32142 let link = ahri_tre_types::DatasetVariableLinkRecord {
32143 dataset_id,
32144 variable_id,
32145 row_role,
32146 };
32147 let mut links = self
32148 .state
32149 .dataset_variables
32150 .lock()
32151 .expect("dataset variable lock should not be poisoned");
32152 if let Some(existing) = links.iter_mut().find(|existing| {
32153 existing.dataset_id == dataset_id && existing.variable_id == variable_id
32154 }) {
32155 *existing = link.clone();
32156 } else {
32157 links.push(link.clone());
32158 }
32159 Ok(link)
32160 }
32161
32162 async fn list_dataset_variables(
32163 &self,
32164 dataset_id: VersionId,
32165 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, ahri_tre_core::CoreError>
32166 {
32167 Ok(self
32168 .state
32169 .dataset_variables
32170 .lock()
32171 .expect("dataset variable lock should not be poisoned")
32172 .iter()
32173 .filter(|link| link.dataset_id == dataset_id)
32174 .cloned()
32175 .collect())
32176 }
32177
32178 async fn list_dataset_variables_for_variable(
32179 &self,
32180 variable_id: VariableId,
32181 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, ahri_tre_core::CoreError>
32182 {
32183 Ok(self
32184 .state
32185 .dataset_variables
32186 .lock()
32187 .expect("dataset variable lock should not be poisoned")
32188 .iter()
32189 .filter(|link| link.variable_id == variable_id)
32190 .cloned()
32191 .collect())
32192 }
32193 }
32194
32195 struct StatefulVocabularyRepository {
32196 state: Arc<AssetWorkflowState>,
32197 }
32198
32199 #[async_trait]
32200 impl VocabularyRepository for StatefulVocabularyRepository {
32201 async fn list_vocabularies(
32202 &self,
32203 domain_id: DomainId,
32204 ) -> Result<Vec<VocabularyRecord>, ahri_tre_core::CoreError> {
32205 let mut vocabularies: Vec<_> = self
32206 .state
32207 .vocabularies_by_id
32208 .lock()
32209 .expect("vocabulary lock should not be poisoned")
32210 .values()
32211 .filter(|vocabulary| vocabulary.domain_id == domain_id)
32212 .cloned()
32213 .collect();
32214 vocabularies.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str()));
32215 Ok(vocabularies)
32216 }
32217
32218 async fn get_vocabulary(
32219 &self,
32220 vocabulary_id: VocabularyId,
32221 ) -> Result<Option<VocabularyRecord>, ahri_tre_core::CoreError> {
32222 Ok(self
32223 .state
32224 .vocabularies_by_id
32225 .lock()
32226 .expect("vocabulary lock should not be poisoned")
32227 .get(&vocabulary_id.0)
32228 .cloned())
32229 }
32230
32231 async fn create_vocabulary(
32232 &self,
32233 vocabulary: &ahri_tre_types::NewVocabularyRecord,
32234 ) -> Result<VocabularyRecord, ahri_tre_core::CoreError> {
32235 if let Some(message) = self
32236 .state
32237 .fail_next_create_vocabulary
32238 .lock()
32239 .expect("vocabulary failure lock should not be poisoned")
32240 .take()
32241 {
32242 return Err(ahri_tre_core::CoreError::Infrastructure(message));
32243 }
32244 let mut vocabularies = self
32245 .state
32246 .vocabularies_by_id
32247 .lock()
32248 .expect("vocabulary lock should not be poisoned");
32249 let vocabulary_id =
32250 VocabularyId(vocabularies.keys().next_back().copied().unwrap_or(0) + 1);
32251 let record = VocabularyRecord {
32252 vocabulary_id,
32253 domain_id: vocabulary.domain_id,
32254 name: vocabulary.name.clone(),
32255 description: vocabulary.description.clone(),
32256 };
32257 vocabularies.insert(record.vocabulary_id.0, record.clone());
32258 Ok(record)
32259 }
32260
32261 async fn list_vocabulary_items(
32262 &self,
32263 vocabulary_id: VocabularyId,
32264 ) -> Result<Vec<VocabularyItemRecord>, ahri_tre_core::CoreError> {
32265 let mut items = self
32266 .state
32267 .vocabulary_items_by_vocabulary
32268 .lock()
32269 .expect("vocabulary item lock should not be poisoned")
32270 .get(&vocabulary_id.0)
32271 .cloned()
32272 .unwrap_or_default();
32273 items.sort_by_key(|item| (item.value, item.code.clone()));
32274 Ok(items)
32275 }
32276
32277 async fn get_vocabulary_item(
32278 &self,
32279 vocabulary_item_id: VocabularyItemId,
32280 ) -> Result<Option<VocabularyItemRecord>, ahri_tre_core::CoreError> {
32281 Ok(self
32282 .state
32283 .vocabulary_items_by_vocabulary
32284 .lock()
32285 .expect("vocabulary item lock should not be poisoned")
32286 .values()
32287 .flat_map(|items| items.iter())
32288 .find(|item| item.vocabulary_item_id == vocabulary_item_id)
32289 .cloned())
32290 }
32291
32292 async fn save_vocabulary(
32293 &self,
32294 vocabulary: &VocabularyRecord,
32295 ) -> Result<VocabularyRecord, ahri_tre_core::CoreError> {
32296 if let Some(message) = self
32297 .state
32298 .fail_next_create_vocabulary
32299 .lock()
32300 .expect("vocabulary failure lock should not be poisoned")
32301 .take()
32302 {
32303 return Err(ahri_tre_core::CoreError::Infrastructure(message));
32304 }
32305 self.state
32306 .vocabularies_by_id
32307 .lock()
32308 .expect("vocabulary lock should not be poisoned")
32309 .insert(vocabulary.vocabulary_id.0, vocabulary.clone());
32310 Ok(vocabulary.clone())
32311 }
32312
32313 async fn delete_vocabulary_record(
32314 &self,
32315 vocabulary_id: VocabularyId,
32316 ) -> Result<bool, ahri_tre_core::CoreError> {
32317 self.state
32318 .vocabulary_items_by_vocabulary
32319 .lock()
32320 .expect("vocabulary item lock should not be poisoned")
32321 .remove(&vocabulary_id.0);
32322 Ok(self
32323 .state
32324 .vocabularies_by_id
32325 .lock()
32326 .expect("vocabulary lock should not be poisoned")
32327 .remove(&vocabulary_id.0)
32328 .is_some())
32329 }
32330
32331 async fn update_vocabulary(
32332 &self,
32333 vocabulary: &VocabularyRecord,
32334 ) -> Result<VocabularyRecord, ahri_tre_core::CoreError> {
32335 self.save_vocabulary(vocabulary).await
32336 }
32337
32338 async fn create_vocabulary_items(
32339 &self,
32340 vocabulary_id: VocabularyId,
32341 items: &[ahri_tre_types::NewVocabularyItemRecord],
32342 ) -> Result<Vec<VocabularyItemRecord>, ahri_tre_core::CoreError> {
32343 let mut by_vocabulary = self
32344 .state
32345 .vocabulary_items_by_vocabulary
32346 .lock()
32347 .expect("vocabulary item lock should not be poisoned");
32348 let next_id = by_vocabulary
32349 .values()
32350 .flat_map(|items| items.iter().map(|item| item.vocabulary_item_id.0))
32351 .max()
32352 .unwrap_or(0)
32353 + 1;
32354 let vocabulary_items = by_vocabulary.entry(vocabulary_id.0).or_default();
32355 let mut created = Vec::with_capacity(items.len());
32356 for (offset, item) in items.iter().enumerate() {
32357 let record = VocabularyItemRecord {
32358 vocabulary_item_id: VocabularyItemId(next_id + offset as i64),
32359 vocabulary_id,
32360 value: item.value,
32361 code: item.code.clone(),
32362 description: item.description.clone(),
32363 };
32364 vocabulary_items.push(record.clone());
32365 created.push(record);
32366 }
32367 Ok(created)
32368 }
32369
32370 async fn save_vocabulary_items(
32371 &self,
32372 items: &[VocabularyItemRecord],
32373 ) -> Result<(), ahri_tre_core::CoreError> {
32374 let mut by_vocabulary = self
32375 .state
32376 .vocabulary_items_by_vocabulary
32377 .lock()
32378 .expect("vocabulary item lock should not be poisoned");
32379 for item in items {
32380 let items = by_vocabulary.entry(item.vocabulary_id.0).or_default();
32381 if let Some(existing) = items
32382 .iter_mut()
32383 .find(|existing| existing.vocabulary_item_id == item.vocabulary_item_id)
32384 {
32385 *existing = item.clone();
32386 } else {
32387 items.push(item.clone());
32388 }
32389 }
32390 Ok(())
32391 }
32392
32393 async fn update_vocabulary_items(
32394 &self,
32395 items: &[VocabularyItemRecord],
32396 ) -> Result<(), ahri_tre_core::CoreError> {
32397 self.save_vocabulary_items(items).await
32398 }
32399
32400 async fn list_vocabulary_mappings(
32401 &self,
32402 ) -> Result<Vec<ahri_tre_types::VocabularyMappingRecord>, ahri_tre_core::CoreError>
32403 {
32404 Ok(self
32405 .state
32406 .vocabulary_mappings
32407 .lock()
32408 .expect("vocabulary mapping lock should not be poisoned")
32409 .clone())
32410 }
32411
32412 async fn create_vocabulary_mapping(
32413 &self,
32414 mapping: &ahri_tre_types::NewVocabularyMappingRecord,
32415 ) -> Result<ahri_tre_types::VocabularyMappingRecord, ahri_tre_core::CoreError> {
32416 let mut mappings = self
32417 .state
32418 .vocabulary_mappings
32419 .lock()
32420 .expect("vocabulary mapping lock should not be poisoned");
32421 if let Some(existing) = mappings.iter().find(|existing| {
32422 existing.from_vocabulary_item == mapping.from_vocabulary_item
32423 && existing.to_vocabulary_item == mapping.to_vocabulary_item
32424 }) {
32425 return Ok(existing.clone());
32426 }
32427 let mapping = ahri_tre_types::VocabularyMappingRecord {
32428 vocabulary_mapping_id: mappings
32429 .iter()
32430 .map(|mapping| mapping.vocabulary_mapping_id)
32431 .max()
32432 .unwrap_or(0)
32433 + 1,
32434 from_vocabulary_item: mapping.from_vocabulary_item,
32435 to_vocabulary_item: mapping.to_vocabulary_item,
32436 };
32437 mappings.push(mapping.clone());
32438 Ok(mapping)
32439 }
32440
32441 async fn save_vocabulary_mapping(
32442 &self,
32443 mapping: &ahri_tre_types::VocabularyMappingRecord,
32444 ) -> Result<ahri_tre_types::VocabularyMappingRecord, ahri_tre_core::CoreError> {
32445 let mut mappings = self
32446 .state
32447 .vocabulary_mappings
32448 .lock()
32449 .expect("vocabulary mapping lock should not be poisoned");
32450 if let Some(existing) = mappings.iter().find(|existing| {
32451 existing.from_vocabulary_item == mapping.from_vocabulary_item
32452 && existing.to_vocabulary_item == mapping.to_vocabulary_item
32453 }) {
32454 return Ok(existing.clone());
32455 }
32456 mappings.push(mapping.clone());
32457 Ok(mapping.clone())
32458 }
32459 }
32460
32461 struct StatefulTagRepository {
32462 state: Arc<AssetWorkflowState>,
32463 }
32464
32465 #[async_trait]
32466 impl TagRepository for StatefulTagRepository {
32467 async fn list_tags(&self) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
32468 let mut tags = self
32469 .state
32470 .tags
32471 .lock()
32472 .expect("tag lock should not be poisoned")
32473 .clone();
32474 tags.sort_by(|left, right| left.name.cmp(&right.name));
32475 Ok(tags)
32476 }
32477
32478 async fn upsert_tag(
32479 &self,
32480 normalized_name: &str,
32481 ) -> Result<ahri_tre_types::TagRecord, CoreError> {
32482 let mut tags = self
32483 .state
32484 .tags
32485 .lock()
32486 .expect("tag lock should not be poisoned");
32487 if let Some(tag) = tags.iter().find(|tag| tag.name == normalized_name) {
32488 return Ok(tag.clone());
32489 }
32490 let tag = ahri_tre_types::TagRecord {
32491 tag_id: ahri_tre_types::TagId(tags.len() as i64 + 1),
32492 name: normalized_name.to_string(),
32493 };
32494 tags.push(tag.clone());
32495 Ok(tag)
32496 }
32497
32498 async fn attach_tag(
32499 &self,
32500 tag_id: ahri_tre_types::TagId,
32501 target: TagAttachmentTarget,
32502 ) -> Result<(), CoreError> {
32503 if let Some(message) = self
32504 .state
32505 .fail_next_attach_tag
32506 .lock()
32507 .expect("tag attachment failure lock should not be poisoned")
32508 .take()
32509 {
32510 return Err(CoreError::Infrastructure(message));
32511 }
32512 let mut links = self
32513 .state
32514 .tag_links
32515 .lock()
32516 .expect("tag link lock should not be poisoned");
32517 if !links.iter().any(|(existing_target, existing_tag_id)| {
32518 existing_target == &target && existing_tag_id == &tag_id
32519 }) {
32520 links.push((target, tag_id));
32521 }
32522 Ok(())
32523 }
32524
32525 async fn detach_tag(
32526 &self,
32527 tag_id: ahri_tre_types::TagId,
32528 target: TagAttachmentTarget,
32529 ) -> Result<(), CoreError> {
32530 self.state
32531 .tag_links
32532 .lock()
32533 .expect("tag link lock should not be poisoned")
32534 .retain(|(existing_target, existing_tag_id)| {
32535 existing_target != &target || existing_tag_id != &tag_id
32536 });
32537 Ok(())
32538 }
32539
32540 async fn delete_tag_if_unused(
32541 &self,
32542 tag_id: ahri_tre_types::TagId,
32543 ) -> Result<bool, CoreError> {
32544 let is_used = self
32545 .state
32546 .tag_links
32547 .lock()
32548 .expect("tag link lock should not be poisoned")
32549 .iter()
32550 .any(|(_, linked_tag_id)| linked_tag_id == &tag_id);
32551 if is_used {
32552 return Ok(false);
32553 }
32554 let mut tags = self
32555 .state
32556 .tags
32557 .lock()
32558 .expect("tag lock should not be poisoned");
32559 let previous_len = tags.len();
32560 tags.retain(|tag| tag.tag_id != tag_id);
32561 Ok(tags.len() != previous_len)
32562 }
32563
32564 async fn list_tags_for_target(
32565 &self,
32566 target: TagAttachmentTarget,
32567 ) -> Result<Vec<ahri_tre_types::TagRecord>, CoreError> {
32568 let tag_ids = self
32569 .state
32570 .tag_links
32571 .lock()
32572 .expect("tag link lock should not be poisoned")
32573 .iter()
32574 .filter(|(existing_target, _)| existing_target == &target)
32575 .map(|(_, tag_id)| *tag_id)
32576 .collect::<Vec<_>>();
32577 let mut tags = self
32578 .state
32579 .tags
32580 .lock()
32581 .expect("tag lock should not be poisoned")
32582 .iter()
32583 .filter(|tag| tag_ids.iter().any(|tag_id| tag_id == &tag.tag_id))
32584 .cloned()
32585 .collect::<Vec<_>>();
32586 tags.sort_by(|left, right| left.name.cmp(&right.name));
32587 Ok(tags)
32588 }
32589 }
32590
32591 #[derive(Default)]
32592 struct TransformationState {
32593 transformations: Mutex<BTreeMap<i64, TransformationRecord>>,
32594 inputs: Mutex<Vec<TransformationInputLinkRecord>>,
32595 outputs: Mutex<Vec<TransformationOutputLinkRecord>>,
32596 fail_next_add_outputs: Mutex<Option<String>>,
32597 }
32598
32599 struct StatefulTransformationRepository {
32600 state: Arc<TransformationState>,
32601 }
32602
32603 #[async_trait]
32604 impl TransformationRepository for StatefulTransformationRepository {
32605 async fn create_transformation(
32606 &self,
32607 transformation: &NewTransformationRecord,
32608 ) -> Result<TransformationRecord, ahri_tre_core::CoreError> {
32609 let mut transformations = self
32610 .state
32611 .transformations
32612 .lock()
32613 .expect("transformation lock should not be poisoned");
32614 let transformation_id = transformations.keys().next_back().copied().unwrap_or(0) + 1;
32615 let persisted = TransformationRecord {
32616 transformation_id: TransformationId(transformation_id),
32617 transformation_type: transformation.transformation_type,
32618 description: transformation.description.clone(),
32619 repository_url: transformation.repository_url.clone(),
32620 commit_hash: transformation.commit_hash.clone(),
32621 file_path: transformation.file_path.clone(),
32622 date_created: transformation.date_created,
32623 created_by: transformation.created_by.clone(),
32624 };
32625 transformations.insert(transformation_id, persisted.clone());
32626 Ok(persisted)
32627 }
32628
32629 async fn save_transformation(
32630 &self,
32631 transformation: &TransformationRecord,
32632 ) -> Result<TransformationRecord, ahri_tre_core::CoreError> {
32633 self.state
32634 .transformations
32635 .lock()
32636 .expect("transformation lock should not be poisoned")
32637 .insert(transformation.transformation_id.0, transformation.clone());
32638 Ok(transformation.clone())
32639 }
32640
32641 async fn add_inputs(
32642 &self,
32643 transformation_id: TransformationId,
32644 input_ids: &[VersionId],
32645 ) -> Result<(), ahri_tre_core::CoreError> {
32646 let mut inputs = self
32647 .state
32648 .inputs
32649 .lock()
32650 .expect("transformation input lock should not be poisoned");
32651 for version_id in input_ids {
32652 let link = TransformationInputLinkRecord {
32653 transformation_id,
32654 version_id: *version_id,
32655 };
32656 if !inputs.iter().any(|existing| existing == &link) {
32657 inputs.push(link);
32658 }
32659 }
32660 Ok(())
32661 }
32662
32663 async fn add_outputs(
32664 &self,
32665 transformation_id: TransformationId,
32666 output_ids: &[VersionId],
32667 ) -> Result<(), ahri_tre_core::CoreError> {
32668 if let Some(message) = self
32669 .state
32670 .fail_next_add_outputs
32671 .lock()
32672 .expect("transformation output failure lock should not be poisoned")
32673 .take()
32674 {
32675 return Err(ahri_tre_core::CoreError::Infrastructure(message));
32676 }
32677 let mut outputs = self
32678 .state
32679 .outputs
32680 .lock()
32681 .expect("transformation output lock should not be poisoned");
32682 for version_id in output_ids {
32683 let link = TransformationOutputLinkRecord {
32684 transformation_id,
32685 version_id: *version_id,
32686 };
32687 if !outputs.iter().any(|existing| existing == &link) {
32688 outputs.push(link);
32689 }
32690 }
32691 Ok(())
32692 }
32693
32694 async fn delete_transformation(
32695 &self,
32696 transformation_id: TransformationId,
32697 ) -> Result<bool, ahri_tre_core::CoreError> {
32698 self.state
32699 .inputs
32700 .lock()
32701 .expect("transformation input lock should not be poisoned")
32702 .retain(|link| link.transformation_id != transformation_id);
32703 self.state
32704 .outputs
32705 .lock()
32706 .expect("transformation output lock should not be poisoned")
32707 .retain(|link| link.transformation_id != transformation_id);
32708 Ok(self
32709 .state
32710 .transformations
32711 .lock()
32712 .expect("transformation lock should not be poisoned")
32713 .remove(&transformation_id.0)
32714 .is_some())
32715 }
32716
32717 async fn list_inputs(
32718 &self,
32719 transformation_id: TransformationId,
32720 ) -> Result<Vec<TransformationInputLinkRecord>, ahri_tre_core::CoreError> {
32721 Ok(self
32722 .state
32723 .inputs
32724 .lock()
32725 .expect("transformation input lock should not be poisoned")
32726 .iter()
32727 .filter(|link| link.transformation_id == transformation_id)
32728 .cloned()
32729 .collect())
32730 }
32731
32732 async fn list_outputs(
32733 &self,
32734 transformation_id: TransformationId,
32735 ) -> Result<Vec<TransformationOutputLinkRecord>, ahri_tre_core::CoreError> {
32736 Ok(self
32737 .state
32738 .outputs
32739 .lock()
32740 .expect("transformation output lock should not be poisoned")
32741 .iter()
32742 .filter(|link| link.transformation_id == transformation_id)
32743 .cloned()
32744 .collect())
32745 }
32746
32747 async fn list_transformations_producing(
32748 &self,
32749 version_id: VersionId,
32750 ) -> Result<Vec<TransformationRecord>, ahri_tre_core::CoreError> {
32751 let transformation_ids: Vec<i64> = self
32752 .state
32753 .outputs
32754 .lock()
32755 .expect("transformation output lock should not be poisoned")
32756 .iter()
32757 .filter(|link| link.version_id == version_id)
32758 .map(|link| link.transformation_id.0)
32759 .collect();
32760 let transformations = self
32761 .state
32762 .transformations
32763 .lock()
32764 .expect("transformation lock should not be poisoned");
32765 Ok(transformation_ids
32766 .iter()
32767 .filter_map(|transformation_id| transformations.get(transformation_id).cloned())
32768 .collect())
32769 }
32770
32771 async fn list_transformations(
32772 &self,
32773 ) -> Result<Vec<TransformationRecord>, ahri_tre_core::CoreError> {
32774 Ok(self
32775 .state
32776 .transformations
32777 .lock()
32778 .expect("transformation lock should not be poisoned")
32779 .values()
32780 .cloned()
32781 .collect())
32782 }
32783 }
32784
32785 struct StatefulDomainRepository {
32786 state: Arc<RegistrationState>,
32787 }
32788
32789 #[async_trait]
32790 impl DomainRepository for StatefulDomainRepository {
32791 async fn list_domains(&self) -> Result<Vec<DomainRecord>, ahri_tre_core::CoreError> {
32792 Ok(self
32793 .state
32794 .domains_by_id
32795 .lock()
32796 .expect("domain lock should not be poisoned")
32797 .values()
32798 .cloned()
32799 .collect())
32800 }
32801
32802 async fn get_domain_by_id(
32803 &self,
32804 domain_id: DomainId,
32805 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
32806 Ok(self
32807 .state
32808 .domains_by_id
32809 .lock()
32810 .expect("domain lock should not be poisoned")
32811 .get(&domain_id.0)
32812 .cloned())
32813 }
32814
32815 async fn get_domain_by_name(
32816 &self,
32817 name: &str,
32818 ) -> Result<Option<DomainRecord>, ahri_tre_core::CoreError> {
32819 Ok(self
32820 .state
32821 .domains_by_name
32822 .lock()
32823 .expect("domain lock should not be poisoned")
32824 .get(name)
32825 .cloned())
32826 }
32827
32828 async fn create_domain(
32829 &self,
32830 domain: &NewDomainRecord,
32831 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
32832 let domain_id = DomainId(
32833 self.state
32834 .domains_by_id
32835 .lock()
32836 .expect("domain lock should not be poisoned")
32837 .keys()
32838 .max()
32839 .copied()
32840 .unwrap_or(0)
32841 + 1,
32842 );
32843 let domain = DomainRecord {
32844 domain_id,
32845 name: domain.name.clone(),
32846 uri: domain.uri.clone(),
32847 description: domain.description.clone(),
32848 };
32849 self.save_domain(&domain).await
32850 }
32851
32852 async fn save_domain(
32853 &self,
32854 domain: &DomainRecord,
32855 ) -> Result<DomainRecord, ahri_tre_core::CoreError> {
32856 self.state
32857 .domains_by_id
32858 .lock()
32859 .expect("domain lock should not be poisoned")
32860 .insert(domain.domain_id.0, domain.clone());
32861 self.state
32862 .domains_by_name
32863 .lock()
32864 .expect("domain lock should not be poisoned")
32865 .insert(domain.name.as_str().to_string(), domain.clone());
32866 Ok(domain.clone())
32867 }
32868
32869 async fn delete_domain_record(
32870 &self,
32871 domain_id: DomainId,
32872 ) -> Result<bool, ahri_tre_core::CoreError> {
32873 let removed = self
32874 .state
32875 .domains_by_id
32876 .lock()
32877 .expect("domain lock should not be poisoned")
32878 .remove(&domain_id.0);
32879 if let Some(domain) = removed.as_ref() {
32880 self.state
32881 .domains_by_name
32882 .lock()
32883 .expect("domain lock should not be poisoned")
32884 .remove(domain.name.as_str());
32885 }
32886 Ok(removed.is_some())
32887 }
32888 }
32889
32890 struct StatefulStudyRepository {
32891 state: Arc<RegistrationState>,
32892 }
32893
32894 #[async_trait]
32895 impl StudyRepository for StatefulStudyRepository {
32896 async fn list_studies(&self) -> Result<Vec<StudyRecord>, ahri_tre_core::CoreError> {
32897 Ok(self
32898 .state
32899 .studies_by_name
32900 .lock()
32901 .expect("study lock should not be poisoned")
32902 .values()
32903 .cloned()
32904 .collect())
32905 }
32906
32907 async fn get_study_by_id(
32908 &self,
32909 study_id: StudyId,
32910 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
32911 Ok(self
32912 .state
32913 .studies_by_id
32914 .lock()
32915 .expect("study lock should not be poisoned")
32916 .get(&study_id.0)
32917 .cloned())
32918 }
32919
32920 async fn get_study_by_name(
32921 &self,
32922 name: &str,
32923 ) -> Result<Option<StudyRecord>, ahri_tre_core::CoreError> {
32924 Ok(self
32925 .state
32926 .studies_by_name
32927 .lock()
32928 .expect("study lock should not be poisoned")
32929 .get(name)
32930 .cloned())
32931 }
32932
32933 async fn create_study(
32934 &self,
32935 study: &NewStudyRecord,
32936 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
32937 let study = StudyRecord {
32938 study_id: StudyId(uuid::Uuid::new_v4()),
32939 name: study.name.clone(),
32940 description: study.description.clone(),
32941 documentation: study.documentation.clone(),
32942 agent_instructions: study.agent_instructions.clone(),
32943 external_id: study.external_id.clone(),
32944 study_type_id: study.study_type_id,
32945 date_created: study.date_created,
32946 created_by: study.created_by.clone(),
32947 };
32948 self.save_study(&study).await
32949 }
32950
32951 async fn save_study(
32952 &self,
32953 study: &StudyRecord,
32954 ) -> Result<StudyRecord, ahri_tre_core::CoreError> {
32955 let mut saved = study.clone();
32956 let creator = saved
32957 .created_by
32958 .get_or_insert_with(|| "test_creator".to_string())
32959 .clone();
32960 self.state
32961 .studies_by_id
32962 .lock()
32963 .expect("study lock should not be poisoned")
32964 .insert(saved.study_id.0, saved.clone());
32965 self.state
32966 .studies_by_name
32967 .lock()
32968 .expect("study lock should not be poisoned")
32969 .insert(saved.name.as_str().to_string(), saved.clone());
32970
32971 if self.state.seed_governance_defaults {
32972 seed_study_governance_defaults(&self.state, saved.study_id, &creator);
32973 }
32974
32975 Ok(saved)
32976 }
32977
32978 async fn delete_study_record(
32979 &self,
32980 study_id: StudyId,
32981 ) -> Result<bool, ahri_tre_core::CoreError> {
32982 let removed = self
32983 .state
32984 .studies_by_id
32985 .lock()
32986 .expect("study lock should not be poisoned")
32987 .remove(&study_id.0);
32988 if let Some(study) = removed.as_ref() {
32989 self.state
32990 .studies_by_name
32991 .lock()
32992 .expect("study lock should not be poisoned")
32993 .remove(study.name.as_str());
32994 }
32995 self.state
32996 .study_domains
32997 .lock()
32998 .expect("study-domain lock should not be poisoned")
32999 .retain(|link| link.study_id != study_id);
33000 self.state
33001 .access_grants
33002 .lock()
33003 .expect("study access lock should not be poisoned")
33004 .retain(|grant| grant.study_id != study_id);
33005 self.state
33006 .custodians
33007 .lock()
33008 .expect("study custodian lock should not be poisoned")
33009 .retain(|custodian| custodian.study_id != study_id);
33010 Ok(removed.is_some())
33011 }
33012 }
33013
33014 struct StatefulStudyDomainRepository {
33015 state: Arc<RegistrationState>,
33016 }
33017
33018 #[async_trait]
33019 impl StudyDomainRepository for StatefulStudyDomainRepository {
33020 async fn link_study_domain(
33021 &self,
33022 study_id: StudyId,
33023 domain_id: DomainId,
33024 ) -> Result<StudyDomainLinkRecord, ahri_tre_core::CoreError> {
33025 let link = StudyDomainLinkRecord {
33026 study_id,
33027 domain_id,
33028 };
33029 let mut links = self
33030 .state
33031 .study_domains
33032 .lock()
33033 .expect("study-domain lock should not be poisoned");
33034 if !links.iter().any(|existing| existing == &link) {
33035 links.push(link.clone());
33036 }
33037 Ok(link)
33038 }
33039
33040 async fn list_domains_for_study(
33041 &self,
33042 study_id: StudyId,
33043 ) -> Result<Vec<DomainRecord>, ahri_tre_core::CoreError> {
33044 let domain_ids: Vec<i64> = self
33045 .state
33046 .study_domains
33047 .lock()
33048 .expect("study-domain lock should not be poisoned")
33049 .iter()
33050 .filter(|link| link.study_id == study_id)
33051 .map(|link| link.domain_id.0)
33052 .collect();
33053 let domains = self
33054 .state
33055 .domains_by_id
33056 .lock()
33057 .expect("domain lock should not be poisoned");
33058 Ok(domain_ids
33059 .iter()
33060 .filter_map(|domain_id| domains.get(domain_id).cloned())
33061 .collect())
33062 }
33063 }
33064
33065 struct StatefulStudyGovernanceRepository {
33066 state: Arc<RegistrationState>,
33067 }
33068
33069 #[async_trait]
33070 impl StudyGovernanceRepository for StatefulStudyGovernanceRepository {
33071 async fn list_access_grants(
33072 &self,
33073 study_id: StudyId,
33074 ) -> Result<Vec<StudyAccessGrantRecord>, ahri_tre_core::CoreError> {
33075 Ok(self
33076 .state
33077 .access_grants
33078 .lock()
33079 .expect("study access lock should not be poisoned")
33080 .iter()
33081 .filter(|grant| grant.study_id == study_id)
33082 .cloned()
33083 .collect())
33084 }
33085
33086 async fn grant_study_access(
33087 &self,
33088 study_id: StudyId,
33089 target_user_id: &str,
33090 ) -> Result<StudyAccessGrantRecord, ahri_tre_core::CoreError> {
33091 let grant = StudyAccessGrantRecord {
33092 study_id,
33093 user_id: target_user_id.to_string(),
33094 date_granted: None,
33095 granted_by: None,
33096 };
33097 self.state
33098 .grant_access_calls
33099 .lock()
33100 .expect("study access grant call lock should not be poisoned")
33101 .push((study_id, target_user_id.to_string()));
33102 self.save_access_grant(&grant).await
33103 }
33104
33105 async fn save_access_grant(
33106 &self,
33107 grant: &StudyAccessGrantRecord,
33108 ) -> Result<StudyAccessGrantRecord, ahri_tre_core::CoreError> {
33109 let mut access_grants = self
33110 .state
33111 .access_grants
33112 .lock()
33113 .expect("study access lock should not be poisoned");
33114 if !access_grants.iter().any(|existing| {
33115 existing.study_id == grant.study_id && existing.user_id == grant.user_id
33116 }) {
33117 access_grants.push(grant.clone());
33118 }
33119 Ok(grant.clone())
33120 }
33121
33122 async fn revoke_access_grant(
33123 &self,
33124 study_id: StudyId,
33125 principal: &str,
33126 ) -> Result<(), ahri_tre_core::CoreError> {
33127 self.revoke_study_access(study_id, principal).await
33128 }
33129
33130 async fn revoke_study_access(
33131 &self,
33132 study_id: StudyId,
33133 target_user_id: &str,
33134 ) -> Result<(), ahri_tre_core::CoreError> {
33135 self.state
33136 .revoke_access_calls
33137 .lock()
33138 .expect("study access revoke call lock should not be poisoned")
33139 .push((study_id, target_user_id.to_string()));
33140 self.state
33141 .access_grants
33142 .lock()
33143 .expect("study access lock should not be poisoned")
33144 .retain(|grant| !(grant.study_id == study_id && grant.user_id == target_user_id));
33145 Ok(())
33146 }
33147
33148 async fn list_custodians(
33149 &self,
33150 study_id: StudyId,
33151 ) -> Result<Vec<StudyCustodianLinkRecord>, ahri_tre_core::CoreError> {
33152 Ok(self
33153 .state
33154 .custodians
33155 .lock()
33156 .expect("study custodian lock should not be poisoned")
33157 .iter()
33158 .filter(|custodian| custodian.study_id == study_id)
33159 .cloned()
33160 .collect())
33161 }
33162
33163 async fn add_delegate_custodian(
33164 &self,
33165 study_id: StudyId,
33166 target_user_id: &str,
33167 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
33168 let custodian = StudyCustodianLinkRecord {
33169 study_id,
33170 user_id: target_user_id.to_string(),
33171 is_primary: false,
33172 date_granted: None,
33173 granted_by: None,
33174 };
33175 self.state
33176 .add_delegate_custodian_calls
33177 .lock()
33178 .expect("delegate custodian call lock should not be poisoned")
33179 .push((study_id, target_user_id.to_string()));
33180 self.save_custodian(&custodian).await
33181 }
33182
33183 async fn transfer_primary_custodianship(
33184 &self,
33185 study_id: StudyId,
33186 target_user_id: &str,
33187 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
33188 self.state
33189 .transfer_primary_custodianship_calls
33190 .lock()
33191 .expect("primary transfer call lock should not be poisoned")
33192 .push((study_id, target_user_id.to_string()));
33193 let mut custodians = self
33194 .state
33195 .custodians
33196 .lock()
33197 .expect("study custodian lock should not be poisoned");
33198 for custodian in custodians
33199 .iter_mut()
33200 .filter(|custodian| custodian.study_id == study_id)
33201 {
33202 custodian.is_primary = custodian.user_id == target_user_id;
33203 }
33204 if !custodians.iter().any(|custodian| {
33205 custodian.study_id == study_id && custodian.user_id == target_user_id
33206 }) {
33207 custodians.push(StudyCustodianLinkRecord {
33208 study_id,
33209 user_id: target_user_id.to_string(),
33210 is_primary: true,
33211 date_granted: None,
33212 granted_by: None,
33213 });
33214 }
33215 Ok(StudyCustodianLinkRecord {
33216 study_id,
33217 user_id: target_user_id.to_string(),
33218 is_primary: true,
33219 date_granted: None,
33220 granted_by: None,
33221 })
33222 }
33223
33224 async fn remove_delegate_custodian(
33225 &self,
33226 study_id: StudyId,
33227 target_user_id: &str,
33228 ) -> Result<(), ahri_tre_core::CoreError> {
33229 self.state
33230 .remove_delegate_custodian_calls
33231 .lock()
33232 .expect("delegate custodian removal call lock should not be poisoned")
33233 .push((study_id, target_user_id.to_string()));
33234 self.state
33235 .custodians
33236 .lock()
33237 .expect("study custodian lock should not be poisoned")
33238 .retain(|custodian| {
33239 !(custodian.study_id == study_id && custodian.user_id == target_user_id)
33240 });
33241 Ok(())
33242 }
33243
33244 async fn save_custodian(
33245 &self,
33246 custodian: &StudyCustodianLinkRecord,
33247 ) -> Result<StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
33248 let mut custodians = self
33249 .state
33250 .custodians
33251 .lock()
33252 .expect("study custodian lock should not be poisoned");
33253 if custodian.is_primary {
33254 for existing in custodians
33255 .iter_mut()
33256 .filter(|existing| existing.study_id == custodian.study_id)
33257 {
33258 existing.is_primary = false;
33259 }
33260 }
33261 if let Some(existing) = custodians.iter_mut().find(|existing| {
33262 existing.study_id == custodian.study_id && existing.user_id == custodian.user_id
33263 }) {
33264 *existing = custodian.clone();
33265 } else {
33266 custodians.push(custodian.clone());
33267 }
33268 Ok(custodian.clone())
33269 }
33270
33271 async fn list_study_duo_restrictions(
33272 &self,
33273 study_id: StudyId,
33274 ) -> Result<Vec<StudyDuoRestrictionRecord>, ahri_tre_core::CoreError> {
33275 Ok(self
33276 .state
33277 .study_duo_restrictions
33278 .lock()
33279 .expect("study DUO restriction lock should not be poisoned")
33280 .iter()
33281 .filter(|restriction| restriction.study_id == study_id)
33282 .cloned()
33283 .collect())
33284 }
33285
33286 async fn create_study_duo_restriction(
33287 &self,
33288 restriction: &NewStudyDuoRestrictionRecord,
33289 ) -> Result<StudyDuoRestrictionRecord, ahri_tre_core::CoreError> {
33290 let mut restrictions = self
33291 .state
33292 .study_duo_restrictions
33293 .lock()
33294 .expect("study DUO restriction lock should not be poisoned");
33295 let restriction_id = restrictions
33296 .iter()
33297 .map(|restriction| restriction.study_duo_restriction_id.0)
33298 .max()
33299 .unwrap_or(0)
33300 + 1;
33301 let persisted = StudyDuoRestrictionRecord {
33302 study_duo_restriction_id: StudyDuoRestrictionId(restriction_id),
33303 study_id: restriction.study_id,
33304 duo_id: restriction.duo_id.clone(),
33305 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
33306 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
33307 qualifier_text: restriction.qualifier_text.clone(),
33308 qualifier_date: restriction.qualifier_date,
33309 qualifier_integer: restriction.qualifier_integer,
33310 };
33311 restrictions.push(persisted.clone());
33312 Ok(persisted)
33313 }
33314
33315 async fn save_study_duo_restriction(
33316 &self,
33317 restriction: &StudyDuoRestrictionRecord,
33318 ) -> Result<StudyDuoRestrictionRecord, ahri_tre_core::CoreError> {
33319 let mut restrictions = self
33320 .state
33321 .study_duo_restrictions
33322 .lock()
33323 .expect("study DUO restriction lock should not be poisoned");
33324 if let Some(existing) = restrictions.iter_mut().find(|existing| {
33325 existing.study_duo_restriction_id == restriction.study_duo_restriction_id
33326 }) {
33327 *existing = restriction.clone();
33328 } else {
33329 restrictions.push(restriction.clone());
33330 }
33331 Ok(restriction.clone())
33332 }
33333
33334 async fn replace_study_duo_restrictions(
33335 &self,
33336 study_id: StudyId,
33337 restrictions: &[NewStudyDuoRestrictionRecord],
33338 ) -> Result<Vec<StudyDuoRestrictionRecord>, ahri_tre_core::CoreError> {
33339 let mut persisted = self
33340 .state
33341 .study_duo_restrictions
33342 .lock()
33343 .expect("study DUO restriction lock should not be poisoned");
33344 persisted.retain(|restriction| restriction.study_id != study_id);
33345 let next_id = persisted
33346 .iter()
33347 .map(|restriction| restriction.study_duo_restriction_id.0)
33348 .max()
33349 .unwrap_or(0)
33350 + 1;
33351 let mut replaced = Vec::with_capacity(restrictions.len());
33352 for (offset, restriction) in restrictions.iter().enumerate() {
33353 let row = StudyDuoRestrictionRecord {
33354 study_duo_restriction_id: StudyDuoRestrictionId(next_id + offset as i64),
33355 study_id,
33356 duo_id: restriction.duo_id.clone(),
33357 qualifier_ontology_namespace: restriction.qualifier_ontology_namespace.clone(),
33358 qualifier_ontology_id: restriction.qualifier_ontology_id.clone(),
33359 qualifier_text: restriction.qualifier_text.clone(),
33360 qualifier_date: restriction.qualifier_date,
33361 qualifier_integer: restriction.qualifier_integer,
33362 };
33363 persisted.push(row.clone());
33364 replaced.push(row);
33365 }
33366 Ok(replaced)
33367 }
33368 }
33369
33370 fn seed_study_governance_defaults(state: &RegistrationState, study_id: StudyId, creator: &str) {
33371 {
33372 let mut custodians = state
33373 .custodians
33374 .lock()
33375 .expect("study custodian lock should not be poisoned");
33376 for existing in custodians
33377 .iter_mut()
33378 .filter(|existing| existing.study_id == study_id)
33379 {
33380 existing.is_primary = false;
33381 }
33382 let primary = StudyCustodianLinkRecord {
33383 study_id,
33384 user_id: creator.to_string(),
33385 is_primary: true,
33386 date_granted: None,
33387 granted_by: Some(creator.to_string()),
33388 };
33389 if let Some(existing) = custodians
33390 .iter_mut()
33391 .find(|existing| existing.study_id == study_id && existing.user_id == creator)
33392 {
33393 *existing = primary;
33394 } else {
33395 custodians.push(primary);
33396 }
33397 }
33398
33399 let mut access_grants = state
33400 .access_grants
33401 .lock()
33402 .expect("study access lock should not be poisoned");
33403 if !access_grants
33404 .iter()
33405 .any(|grant| grant.study_id == study_id && grant.user_id == creator)
33406 {
33407 access_grants.push(StudyAccessGrantRecord {
33408 study_id,
33409 user_id: creator.to_string(),
33410 date_granted: None,
33411 granted_by: Some(creator.to_string()),
33412 });
33413 }
33414 }
33415
33416 fn test_domain(domain_id: i64, name: &str) -> DomainRecord {
33417 DomainRecord {
33418 domain_id: DomainId(domain_id),
33419 name: nc_name(name),
33420 description: Some("P1.10 fixture domain".to_string()),
33421 uri: Some(format!("https://example.test/domains/{name}")),
33422 }
33423 }
33424
33425 fn test_new_domain(name: &str) -> NewDomainRecord {
33426 NewDomainRecord {
33427 name: nc_name(name),
33428 description: Some("P1.10 new domain".to_string()),
33429 uri: Some(format!("https://example.test/domains/{name}")),
33430 }
33431 }
33432
33433 fn test_study(name: &str) -> StudyRecord {
33434 StudyRecord {
33435 study_id: StudyId(uuid::Uuid::new_v4()),
33436 name: nc_name(name),
33437 description: Some("P1.10 fixture study".to_string()),
33438 documentation: Some("P1.10 fixture documentation".to_string()),
33439 agent_instructions: Some("P1.10 fixture instructions".to_string()),
33440 external_id: Some(format!("external-{name}")),
33441 study_type_id: None,
33442 date_created: None,
33443 created_by: Some("test_creator".to_string()),
33444 }
33445 }
33446
33447 fn test_new_study(name: &str) -> NewStudyRecord {
33448 NewStudyRecord {
33449 name: nc_name(name),
33450 description: Some("P1.10 new study".to_string()),
33451 documentation: Some("P1.10 new documentation".to_string()),
33452 agent_instructions: Some("P1.10 new instructions".to_string()),
33453 external_id: Some(format!("external-{name}")),
33454 study_type_id: None,
33455 date_created: None,
33456 created_by: Some("test_creator".to_string()),
33457 }
33458 }
33459
33460 fn test_asset(study_id: StudyId, name: &str) -> AssetRecord {
33461 AssetRecord {
33462 asset_id: AssetId(uuid::Uuid::new_v4()),
33463 study_id,
33464 name: nc_name(name),
33465 description: Some("P1.11 fixture asset".to_string()),
33466 agent_instructions: Some("P1.11 fixture asset instructions".to_string()),
33467 asset_type: AssetType::Dataset,
33468 date_created: None,
33469 created_by: Some("test_creator".to_string()),
33470 }
33471 }
33472
33473 fn test_asset_version(
33474 asset_id: AssetId,
33475 major: i32,
33476 minor: i32,
33477 patch: i32,
33478 version_note: &str,
33479 ) -> AssetVersionRecord {
33480 AssetVersionRecord {
33481 version_id: VersionId(uuid::Uuid::new_v4()),
33482 asset_id,
33483 major,
33484 minor,
33485 patch,
33486 version_label: None,
33487 version_note: Some(version_note.to_string()),
33488 is_latest: None,
33489 doi: None,
33490 created_at: None,
33491 created_by: Some("test_creator".to_string()),
33492 }
33493 }
33494
33495 fn seed_completed_dataset_version(
33496 repositories: &AssetWorkflowRepositories,
33497 asset: &AssetRecord,
33498 major: i32,
33499 minor: i32,
33500 patch: i32,
33501 is_latest: bool,
33502 version_note: &str,
33503 ) -> AssetVersionRecord {
33504 let _ = block_on(repositories.assets.save_asset(asset)).expect("dataset asset should save");
33505 let version = AssetVersionRecord {
33506 is_latest: Some(is_latest),
33507 ..test_asset_version(asset.asset_id, major, minor, patch, version_note)
33508 };
33509 let version = block_on(repositories.assets.save_asset_version(&version))
33510 .expect("dataset asset version should save");
33511 let _ = block_on(repositories.assets.save_dataset(&DatasetRecord {
33512 dataset_id: version.version_id,
33513 }))
33514 .expect("dataset subtype should save");
33515 let transformation = test_transformation(
33516 900_000 + i64::from(major) * 10_000 + i64::from(minor) * 100 + i64::from(patch),
33517 TransformationType::Ingest,
33518 "completed Dataset fixture",
33519 );
33520 let transformation = block_on(
33521 repositories
33522 .transformations
33523 .create_transformation(&transformation),
33524 )
33525 .unwrap();
33526 block_on(
33527 repositories
33528 .transformations
33529 .add_outputs(transformation.transformation_id, &[version.version_id]),
33530 )
33531 .unwrap();
33532 version
33533 }
33534
33535 fn seed_completed_datafile_version(
33536 repositories: &AssetWorkflowRepositories,
33537 lake_root: &Path,
33538 asset: &AssetRecord,
33539 version_number: (i32, i32, i32),
33540 is_latest: bool,
33541 contents: &[u8],
33542 ) -> AssetVersionRecord {
33543 let (major, minor, patch) = version_number;
33544 let mut asset = asset.clone();
33545 asset.asset_type = AssetType::File;
33546 let _ = block_on(repositories.assets.save_asset(&asset)).expect("file asset should save");
33547 let version = AssetVersionRecord {
33548 is_latest: Some(is_latest),
33549 ..test_asset_version(asset.asset_id, major, minor, patch, "file version")
33550 };
33551 let version = block_on(repositories.assets.save_asset_version(&version))
33552 .expect("file asset version should save");
33553 let mut source = Cursor::new(contents.to_vec());
33554 let staged = DuckLakeAdapter::new(lake_root.display().to_string())
33555 .stage_datafile_stream(
33556 &StageDataFileStreamRequest {
33557 source_file_name: format!("{}.txt", asset.name.as_str()),
33558 asset_id: asset.asset_id,
33559 version_id: version.version_id,
33560 study_id: asset.study_id,
33561 asset_name: asset.name.clone(),
33562 major,
33563 minor,
33564 patch,
33565 edam_format: "text/plain".to_string(),
33566 compression: DataFileCompression::None,
33567 encryption: DataFileEncryption::None,
33568 },
33569 &mut source,
33570 )
33571 .expect("test datafile should stage");
33572 block_on(repositories.assets.save_datafile(&staged.datafile))
33573 .expect("datafile metadata should save");
33574 version
33575 }
33576
33577 fn test_datafile(version_id: VersionId) -> DataFileRecord {
33578 DataFileRecord {
33579 datafile_id: version_id,
33580 size_bytes: None,
33581 compressed: Some(false),
33582 encrypted: Some(false),
33583 compression_algorithm: None,
33584 encryption_algorithm: None,
33585 encryption_key: None,
33586 storage_uri: format!("lake://{}", version_id.0),
33587 edam_format: "EDAM:format_3752".to_string(),
33588 digest: ahri_tre_types::Sha256Hex::parse("a".repeat(64))
33589 .expect("fixture digest should be valid"),
33590 }
33591 }
33592
33593 fn test_variable(
33594 variable_id: i64,
33595 domain_id: i64,
33596 name: &str,
33597 key_role: KeyRole,
33598 vocabulary_id: Option<VocabularyId>,
33599 ) -> VariableRecord {
33600 VariableRecord {
33601 variable_id: VariableId(variable_id),
33602 domain_id: DomainId(domain_id),
33603 name: nc_name(name),
33604 value_type_id: ValueTypeId(3),
33605 value_format: None,
33606 vocabulary_id,
33607 key_role,
33608 description: Some(format!("Variable {name}")),
33609 note: None,
33610 ontology_namespace: None,
33611 ontology_class: None,
33612 }
33613 }
33614
33615 fn test_vocabulary(vocabulary_id: i64, domain_id: i64, name: &str) -> VocabularyRecord {
33616 VocabularyRecord {
33617 vocabulary_id: VocabularyId(vocabulary_id),
33618 domain_id: DomainId(domain_id),
33619 name: nc_name(name),
33620 description: Some(format!("Vocabulary {name}")),
33621 }
33622 }
33623
33624 fn test_vocabulary_item(
33625 item_id: i64,
33626 vocabulary_id: i64,
33627 value: i32,
33628 code: &str,
33629 ) -> VocabularyItemRecord {
33630 VocabularyItemRecord {
33631 vocabulary_item_id: VocabularyItemId(item_id),
33632 vocabulary_id: VocabularyId(vocabulary_id),
33633 value,
33634 code: code.to_string(),
33635 description: Some(format!("Code {code}")),
33636 }
33637 }
33638
33639 fn test_transformation(
33640 _transformation_id: i64,
33641 transformation_type: TransformationType,
33642 description: &str,
33643 ) -> NewTransformationRecord {
33644 NewTransformationRecord {
33645 transformation_type,
33646 description: description.to_string(),
33647 repository_url: Some("https://example.test/ahri-tre-rs.git".to_string()),
33648 commit_hash: Some("abcdef1".to_string()),
33649 file_path: Some("transforms/example.sql".to_string()),
33650 date_created: None,
33651 created_by: Some("test_creator".to_string()),
33652 }
33653 }
33654
33655 fn test_transformation_without_source(
33656 _transformation_id: i64,
33657 transformation_type: TransformationType,
33658 description: &str,
33659 ) -> NewTransformationRecord {
33660 NewTransformationRecord {
33661 transformation_type,
33662 description: description.to_string(),
33663 repository_url: None,
33664 commit_hash: None,
33665 file_path: None,
33666 date_created: None,
33667 created_by: Some("test_creator".to_string()),
33668 }
33669 }
33670
33671 #[test]
33672 fn synthetic_redcap_repeat_columns_are_shortened_without_collisions() {
33673 let dataset_name = "redcap_797_dispense_emergency_contraceptives";
33674
33675 let instrument = redcap_synthetic_output_column(dataset_name, "redcap_repeat_instrument");
33676 let instance = redcap_synthetic_output_column(dataset_name, "redcap_repeat_instance");
33677
33678 assert_ne!(instrument, instance);
33679 assert!(instrument.len() <= 63);
33680 assert!(instance.len() <= 63);
33681 assert!(instrument.contains("redcap_repeat_instrument"));
33682 assert!(instance.contains("redcap_repeat_instance"));
33683 }
33684
33685 fn assert_creator_seeded_as_access_grant_and_primary_custodian(
33686 repositories: &RegistrationRepositories,
33687 study_id: StudyId,
33688 creator: &str,
33689 ) {
33690 assert!(
33691 repositories
33692 .state()
33693 .access_grants
33694 .lock()
33695 .expect("study access lock should not be poisoned")
33696 .iter()
33697 .any(|grant| grant.study_id == study_id && grant.user_id == creator),
33698 "creator should be seeded into study_access"
33699 );
33700 assert!(
33701 repositories
33702 .state()
33703 .custodians
33704 .lock()
33705 .expect("study custodian lock should not be poisoned")
33706 .iter()
33707 .any(|custodian| custodian.study_id == study_id
33708 && custodian.user_id == creator
33709 && custodian.is_primary),
33710 "creator should be seeded as primary study custodian"
33711 );
33712 }
33713
33714 fn nc_name(value: &str) -> NcName {
33715 NcName::parse(value).expect("fixture NCName should validate")
33716 }
33717
33718 fn block_on<F: std::future::Future>(future: F) -> F::Output {
33719 tokio::runtime::Builder::new_current_thread()
33720 .enable_all()
33721 .build()
33722 .expect("test Tokio runtime should build")
33723 .block_on(future)
33724 }
33725
33726 struct NoopDomainRepository;
33727 struct NoopStudyRepository;
33728 struct NoopStudyDomainRepository;
33729 struct NoopStudyGovernanceRepository;
33730 struct NoopAssetRepository;
33731 struct NoopVariableRepository;
33732 struct NoopVocabularyRepository;
33733 struct NoopEntityRepository;
33734 struct NoopTransformationRepository;
33735
33736 #[async_trait]
33737 impl DomainRepository for NoopDomainRepository {
33738 async fn list_domains(
33739 &self,
33740 ) -> Result<Vec<ahri_tre_types::DomainRecord>, ahri_tre_core::CoreError> {
33741 Ok(Vec::new())
33742 }
33743
33744 async fn get_domain_by_id(
33745 &self,
33746 _domain_id: ahri_tre_types::DomainId,
33747 ) -> Result<Option<ahri_tre_types::DomainRecord>, ahri_tre_core::CoreError> {
33748 panic!("test repository should not be called")
33749 }
33750
33751 async fn get_domain_by_name(
33752 &self,
33753 _name: &str,
33754 ) -> Result<Option<ahri_tre_types::DomainRecord>, ahri_tre_core::CoreError> {
33755 panic!("test repository should not be called")
33756 }
33757
33758 async fn create_domain(
33759 &self,
33760 _domain: &ahri_tre_types::NewDomainRecord,
33761 ) -> Result<ahri_tre_types::DomainRecord, ahri_tre_core::CoreError> {
33762 panic!("test repository should not be called")
33763 }
33764
33765 async fn save_domain(
33766 &self,
33767 _domain: &ahri_tre_types::DomainRecord,
33768 ) -> Result<ahri_tre_types::DomainRecord, ahri_tre_core::CoreError> {
33769 panic!("test repository should not be called")
33770 }
33771 }
33772
33773 #[async_trait]
33774 impl StudyRepository for NoopStudyRepository {
33775 async fn list_studies(
33776 &self,
33777 ) -> Result<Vec<ahri_tre_types::StudyRecord>, ahri_tre_core::CoreError> {
33778 Ok(Vec::new())
33779 }
33780
33781 async fn get_study_by_id(
33782 &self,
33783 _study_id: ahri_tre_types::StudyId,
33784 ) -> Result<Option<ahri_tre_types::StudyRecord>, ahri_tre_core::CoreError> {
33785 panic!("test repository should not be called")
33786 }
33787
33788 async fn get_study_by_name(
33789 &self,
33790 _name: &str,
33791 ) -> Result<Option<ahri_tre_types::StudyRecord>, ahri_tre_core::CoreError> {
33792 panic!("test repository should not be called")
33793 }
33794
33795 async fn create_study(
33796 &self,
33797 _study: &ahri_tre_types::NewStudyRecord,
33798 ) -> Result<ahri_tre_types::StudyRecord, ahri_tre_core::CoreError> {
33799 panic!("test repository should not be called")
33800 }
33801
33802 async fn save_study(
33803 &self,
33804 _study: &ahri_tre_types::StudyRecord,
33805 ) -> Result<ahri_tre_types::StudyRecord, ahri_tre_core::CoreError> {
33806 panic!("test repository should not be called")
33807 }
33808 }
33809
33810 #[async_trait]
33811 impl StudyDomainRepository for NoopStudyDomainRepository {
33812 async fn link_study_domain(
33813 &self,
33814 _study_id: ahri_tre_types::StudyId,
33815 _domain_id: ahri_tre_types::DomainId,
33816 ) -> Result<ahri_tre_types::StudyDomainLinkRecord, ahri_tre_core::CoreError> {
33817 panic!("test repository should not be called")
33818 }
33819
33820 async fn list_domains_for_study(
33821 &self,
33822 _study_id: ahri_tre_types::StudyId,
33823 ) -> Result<Vec<ahri_tre_types::DomainRecord>, ahri_tre_core::CoreError> {
33824 panic!("test repository should not be called")
33825 }
33826 }
33827
33828 #[async_trait]
33829 impl StudyGovernanceRepository for NoopStudyGovernanceRepository {
33830 async fn list_access_grants(
33831 &self,
33832 _study_id: ahri_tre_types::StudyId,
33833 ) -> Result<Vec<ahri_tre_types::StudyAccessGrantRecord>, ahri_tre_core::CoreError> {
33834 panic!("test repository should not be called")
33835 }
33836
33837 async fn save_access_grant(
33838 &self,
33839 _grant: &ahri_tre_types::StudyAccessGrantRecord,
33840 ) -> Result<ahri_tre_types::StudyAccessGrantRecord, ahri_tre_core::CoreError> {
33841 panic!("test repository should not be called")
33842 }
33843
33844 async fn revoke_access_grant(
33845 &self,
33846 _study_id: ahri_tre_types::StudyId,
33847 _principal: &str,
33848 ) -> Result<(), ahri_tre_core::CoreError> {
33849 panic!("test repository should not be called")
33850 }
33851
33852 async fn revoke_study_access(
33853 &self,
33854 _study_id: ahri_tre_types::StudyId,
33855 _target_user_id: &str,
33856 ) -> Result<(), ahri_tre_core::CoreError> {
33857 panic!("test repository should not be called")
33858 }
33859
33860 async fn list_custodians(
33861 &self,
33862 _study_id: ahri_tre_types::StudyId,
33863 ) -> Result<Vec<ahri_tre_types::StudyCustodianLinkRecord>, ahri_tre_core::CoreError>
33864 {
33865 panic!("test repository should not be called")
33866 }
33867
33868 async fn transfer_primary_custodianship(
33869 &self,
33870 _study_id: ahri_tre_types::StudyId,
33871 _target_user_id: &str,
33872 ) -> Result<ahri_tre_types::StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
33873 panic!("test repository should not be called")
33874 }
33875
33876 async fn save_custodian(
33877 &self,
33878 _custodian: &ahri_tre_types::StudyCustodianLinkRecord,
33879 ) -> Result<ahri_tre_types::StudyCustodianLinkRecord, ahri_tre_core::CoreError> {
33880 panic!("test repository should not be called")
33881 }
33882 }
33883
33884 #[async_trait]
33885 impl AssetRepository for NoopAssetRepository {
33886 async fn list_assets_for_study(
33887 &self,
33888 _study_id: ahri_tre_types::StudyId,
33889 ) -> Result<Vec<ahri_tre_types::AssetRecord>, ahri_tre_core::CoreError> {
33890 panic!("test repository should not be called")
33891 }
33892
33893 async fn get_asset_by_id(
33894 &self,
33895 _asset_id: ahri_tre_types::AssetId,
33896 ) -> Result<Option<ahri_tre_types::AssetRecord>, ahri_tre_core::CoreError> {
33897 panic!("test repository should not be called")
33898 }
33899
33900 async fn get_asset_by_name(
33901 &self,
33902 _study_id: ahri_tre_types::StudyId,
33903 _name: &str,
33904 ) -> Result<Option<ahri_tre_types::AssetRecord>, ahri_tre_core::CoreError> {
33905 panic!("test repository should not be called")
33906 }
33907
33908 async fn create_asset(
33909 &self,
33910 _asset: &ahri_tre_types::NewAssetRecord,
33911 ) -> Result<ahri_tre_types::AssetRecord, ahri_tre_core::CoreError> {
33912 panic!("test repository should not be called")
33913 }
33914
33915 async fn save_asset(
33916 &self,
33917 _asset: &ahri_tre_types::AssetRecord,
33918 ) -> Result<ahri_tre_types::AssetRecord, ahri_tre_core::CoreError> {
33919 panic!("test repository should not be called")
33920 }
33921
33922 async fn list_asset_versions(
33923 &self,
33924 _asset_id: ahri_tre_types::AssetId,
33925 ) -> Result<Vec<ahri_tre_types::AssetVersionRecord>, ahri_tre_core::CoreError> {
33926 panic!("test repository should not be called")
33927 }
33928
33929 async fn get_asset_version(
33930 &self,
33931 _version_id: ahri_tre_types::VersionId,
33932 ) -> Result<Option<ahri_tre_types::AssetVersionRecord>, ahri_tre_core::CoreError> {
33933 panic!("test repository should not be called")
33934 }
33935
33936 async fn create_asset_version(
33937 &self,
33938 _version: &ahri_tre_types::NewAssetVersionRecord,
33939 ) -> Result<ahri_tre_types::AssetVersionRecord, ahri_tre_core::CoreError> {
33940 panic!("test repository should not be called")
33941 }
33942
33943 async fn save_asset_version(
33944 &self,
33945 _version: &ahri_tre_types::AssetVersionRecord,
33946 ) -> Result<ahri_tre_types::AssetVersionRecord, ahri_tre_core::CoreError> {
33947 panic!("test repository should not be called")
33948 }
33949
33950 async fn list_datasets_for_asset(
33951 &self,
33952 _asset_id: ahri_tre_types::AssetId,
33953 ) -> Result<Vec<ahri_tre_types::DatasetRecord>, ahri_tre_core::CoreError> {
33954 panic!("test repository should not be called")
33955 }
33956
33957 async fn get_dataset(
33958 &self,
33959 _dataset_id: ahri_tre_types::VersionId,
33960 ) -> Result<Option<ahri_tre_types::DatasetRecord>, ahri_tre_core::CoreError> {
33961 panic!("test repository should not be called")
33962 }
33963
33964 async fn save_dataset(
33965 &self,
33966 _dataset: &ahri_tre_types::DatasetRecord,
33967 ) -> Result<ahri_tre_types::DatasetRecord, ahri_tre_core::CoreError> {
33968 panic!("test repository should not be called")
33969 }
33970
33971 async fn list_datafiles_for_asset(
33972 &self,
33973 _asset_id: ahri_tre_types::AssetId,
33974 ) -> Result<Vec<ahri_tre_types::DataFileRecord>, ahri_tre_core::CoreError> {
33975 panic!("test repository should not be called")
33976 }
33977
33978 async fn get_datafile(
33979 &self,
33980 _datafile_id: ahri_tre_types::VersionId,
33981 ) -> Result<Option<ahri_tre_types::DataFileRecord>, ahri_tre_core::CoreError> {
33982 panic!("test repository should not be called")
33983 }
33984
33985 async fn save_datafile(
33986 &self,
33987 _datafile: &ahri_tre_types::DataFileRecord,
33988 ) -> Result<ahri_tre_types::DataFileRecord, ahri_tre_core::CoreError> {
33989 panic!("test repository should not be called")
33990 }
33991 }
33992
33993 #[async_trait]
33994 impl VariableRepository for NoopVariableRepository {
33995 async fn list_value_types(
33996 &self,
33997 ) -> Result<Vec<ahri_tre_types::ValueTypeRecord>, ahri_tre_core::CoreError> {
33998 panic!("test repository should not be called")
33999 }
34000
34001 async fn list_domain_variables(
34002 &self,
34003 _domain_id: ahri_tre_types::DomainId,
34004 ) -> Result<Vec<ahri_tre_types::VariableRecord>, ahri_tre_core::CoreError> {
34005 panic!("test repository should not be called")
34006 }
34007
34008 async fn get_variable(
34009 &self,
34010 _variable_id: ahri_tre_types::VariableId,
34011 ) -> Result<Option<ahri_tre_types::VariableRecord>, ahri_tre_core::CoreError> {
34012 panic!("test repository should not be called")
34013 }
34014
34015 async fn create_variable(
34016 &self,
34017 _variable: &ahri_tre_types::NewVariableRecord,
34018 ) -> Result<ahri_tre_types::VariableRecord, ahri_tre_core::CoreError> {
34019 panic!("test repository should not be called")
34020 }
34021
34022 async fn save_variable(
34023 &self,
34024 _variable: &ahri_tre_types::VariableRecord,
34025 ) -> Result<ahri_tre_types::VariableRecord, ahri_tre_core::CoreError> {
34026 panic!("test repository should not be called")
34027 }
34028
34029 async fn update_variable_metadata(
34030 &self,
34031 _variable: &ahri_tre_types::VariableRecord,
34032 ) -> Result<ahri_tre_types::VariableRecord, ahri_tre_core::CoreError> {
34033 panic!("test repository should not be called")
34034 }
34035
34036 async fn link_dataset_variable(
34037 &self,
34038 _dataset_id: ahri_tre_types::VersionId,
34039 _variable_id: ahri_tre_types::VariableId,
34040 _row_role: ahri_tre_types::KeyRole,
34041 ) -> Result<ahri_tre_types::DatasetVariableLinkRecord, ahri_tre_core::CoreError> {
34042 panic!("test repository should not be called")
34043 }
34044
34045 async fn list_dataset_variables(
34046 &self,
34047 _dataset_id: ahri_tre_types::VersionId,
34048 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, ahri_tre_core::CoreError>
34049 {
34050 panic!("test repository should not be called")
34051 }
34052
34053 async fn list_dataset_variables_for_variable(
34054 &self,
34055 _variable_id: ahri_tre_types::VariableId,
34056 ) -> Result<Vec<ahri_tre_types::DatasetVariableLinkRecord>, ahri_tre_core::CoreError>
34057 {
34058 panic!("test repository should not be called")
34059 }
34060 }
34061
34062 #[async_trait]
34063 impl VocabularyRepository for NoopVocabularyRepository {
34064 async fn list_vocabularies(
34065 &self,
34066 _domain_id: ahri_tre_types::DomainId,
34067 ) -> Result<Vec<ahri_tre_types::VocabularyRecord>, ahri_tre_core::CoreError> {
34068 panic!("test repository should not be called")
34069 }
34070
34071 async fn get_vocabulary(
34072 &self,
34073 _vocabulary_id: ahri_tre_types::VocabularyId,
34074 ) -> Result<Option<ahri_tre_types::VocabularyRecord>, ahri_tre_core::CoreError> {
34075 panic!("test repository should not be called")
34076 }
34077
34078 async fn create_vocabulary(
34079 &self,
34080 _vocabulary: &ahri_tre_types::NewVocabularyRecord,
34081 ) -> Result<ahri_tre_types::VocabularyRecord, ahri_tre_core::CoreError> {
34082 panic!("test repository should not be called")
34083 }
34084
34085 async fn save_vocabulary(
34086 &self,
34087 _vocabulary: &ahri_tre_types::VocabularyRecord,
34088 ) -> Result<ahri_tre_types::VocabularyRecord, ahri_tre_core::CoreError> {
34089 panic!("test repository should not be called")
34090 }
34091
34092 async fn update_vocabulary(
34093 &self,
34094 _vocabulary: &ahri_tre_types::VocabularyRecord,
34095 ) -> Result<ahri_tre_types::VocabularyRecord, ahri_tre_core::CoreError> {
34096 panic!("test repository should not be called")
34097 }
34098
34099 async fn list_vocabulary_items(
34100 &self,
34101 _vocabulary_id: ahri_tre_types::VocabularyId,
34102 ) -> Result<Vec<ahri_tre_types::VocabularyItemRecord>, ahri_tre_core::CoreError> {
34103 panic!("test repository should not be called")
34104 }
34105
34106 async fn get_vocabulary_item(
34107 &self,
34108 _vocabulary_item_id: ahri_tre_types::VocabularyItemId,
34109 ) -> Result<Option<ahri_tre_types::VocabularyItemRecord>, ahri_tre_core::CoreError>
34110 {
34111 panic!("test repository should not be called")
34112 }
34113
34114 async fn create_vocabulary_items(
34115 &self,
34116 _vocabulary_id: ahri_tre_types::VocabularyId,
34117 _items: &[ahri_tre_types::NewVocabularyItemRecord],
34118 ) -> Result<Vec<ahri_tre_types::VocabularyItemRecord>, ahri_tre_core::CoreError> {
34119 panic!("test repository should not be called")
34120 }
34121
34122 async fn save_vocabulary_items(
34123 &self,
34124 _items: &[ahri_tre_types::VocabularyItemRecord],
34125 ) -> Result<(), ahri_tre_core::CoreError> {
34126 panic!("test repository should not be called")
34127 }
34128
34129 async fn update_vocabulary_items(
34130 &self,
34131 _items: &[ahri_tre_types::VocabularyItemRecord],
34132 ) -> Result<(), ahri_tre_core::CoreError> {
34133 panic!("test repository should not be called")
34134 }
34135
34136 async fn list_vocabulary_mappings(
34137 &self,
34138 ) -> Result<Vec<ahri_tre_types::VocabularyMappingRecord>, ahri_tre_core::CoreError>
34139 {
34140 panic!("test repository should not be called")
34141 }
34142
34143 async fn create_vocabulary_mapping(
34144 &self,
34145 _mapping: &ahri_tre_types::NewVocabularyMappingRecord,
34146 ) -> Result<ahri_tre_types::VocabularyMappingRecord, ahri_tre_core::CoreError> {
34147 panic!("test repository should not be called")
34148 }
34149
34150 async fn save_vocabulary_mapping(
34151 &self,
34152 _mapping: &ahri_tre_types::VocabularyMappingRecord,
34153 ) -> Result<ahri_tre_types::VocabularyMappingRecord, ahri_tre_core::CoreError> {
34154 panic!("test repository should not be called")
34155 }
34156 }
34157
34158 #[async_trait]
34159 impl EntityRepository for NoopEntityRepository {
34160 async fn create_entity(
34161 &self,
34162 _entity: &ahri_tre_types::NewEntityRecord,
34163 ) -> Result<ahri_tre_types::EntityRecord, ahri_tre_core::CoreError> {
34164 panic!("test repository should not be called")
34165 }
34166
34167 async fn save_entity(
34168 &self,
34169 _entity: &ahri_tre_types::EntityRecord,
34170 ) -> Result<ahri_tre_types::EntityRecord, ahri_tre_core::CoreError> {
34171 panic!("test repository should not be called")
34172 }
34173
34174 async fn create_entity_relation(
34175 &self,
34176 _relation: &ahri_tre_types::NewEntityRelationRecord,
34177 ) -> Result<ahri_tre_types::EntityRelationRecord, ahri_tre_core::CoreError> {
34178 panic!("test repository should not be called")
34179 }
34180
34181 async fn save_entity_relation(
34182 &self,
34183 _relation: &ahri_tre_types::EntityRelationRecord,
34184 ) -> Result<ahri_tre_types::EntityRelationRecord, ahri_tre_core::CoreError> {
34185 panic!("test repository should not be called")
34186 }
34187 }
34188
34189 #[async_trait]
34190 impl TransformationRepository for NoopTransformationRepository {
34191 async fn create_transformation(
34192 &self,
34193 transformation: &ahri_tre_types::NewTransformationRecord,
34194 ) -> Result<ahri_tre_types::TransformationRecord, ahri_tre_core::CoreError> {
34195 Ok(ahri_tre_types::TransformationRecord {
34196 transformation_id: ahri_tre_types::TransformationId(1),
34197 transformation_type: transformation.transformation_type,
34198 description: transformation.description.clone(),
34199 repository_url: transformation.repository_url.clone(),
34200 commit_hash: transformation.commit_hash.clone(),
34201 file_path: transformation.file_path.clone(),
34202 date_created: transformation.date_created,
34203 created_by: transformation.created_by.clone(),
34204 })
34205 }
34206
34207 async fn save_transformation(
34208 &self,
34209 _transformation: &ahri_tre_types::TransformationRecord,
34210 ) -> Result<ahri_tre_types::TransformationRecord, ahri_tre_core::CoreError> {
34211 panic!("test repository should not be called")
34212 }
34213
34214 async fn add_inputs(
34215 &self,
34216 _transformation_id: ahri_tre_types::TransformationId,
34217 _input_ids: &[ahri_tre_types::VersionId],
34218 ) -> Result<(), ahri_tre_core::CoreError> {
34219 panic!("test repository should not be called")
34220 }
34221
34222 async fn add_outputs(
34223 &self,
34224 _transformation_id: ahri_tre_types::TransformationId,
34225 _output_ids: &[ahri_tre_types::VersionId],
34226 ) -> Result<(), ahri_tre_core::CoreError> {
34227 panic!("test repository should not be called")
34228 }
34229
34230 async fn list_inputs(
34231 &self,
34232 _transformation_id: ahri_tre_types::TransformationId,
34233 ) -> Result<Vec<ahri_tre_types::TransformationInputLinkRecord>, ahri_tre_core::CoreError>
34234 {
34235 panic!("test repository should not be called")
34236 }
34237
34238 async fn list_outputs(
34239 &self,
34240 _transformation_id: ahri_tre_types::TransformationId,
34241 ) -> Result<Vec<ahri_tre_types::TransformationOutputLinkRecord>, ahri_tre_core::CoreError>
34242 {
34243 panic!("test repository should not be called")
34244 }
34245
34246 async fn list_transformations_producing(
34247 &self,
34248 _version_id: ahri_tre_types::VersionId,
34249 ) -> Result<Vec<ahri_tre_types::TransformationRecord>, ahri_tre_core::CoreError> {
34250 panic!("test repository should not be called")
34251 }
34252
34253 async fn list_transformations(
34254 &self,
34255 ) -> Result<Vec<ahri_tre_types::TransformationRecord>, ahri_tre_core::CoreError> {
34256 Ok(Vec::new())
34257 }
34258 }
34259}