diff --git a/docs/guides/extensions/curator/metadata_contribution.md b/docs/guides/extensions/curator/metadata_contribution.md index 833b196a9..cc82d9e60 100644 --- a/docs/guides/extensions/curator/metadata_contribution.md +++ b/docs/guides/extensions/curator/metadata_contribution.md @@ -7,7 +7,8 @@ This guide shows how to programmatically complete a record-based metadata curati By following this guide, you will: - List curation tasks in a Synapse project -- Get or create a Grid session for a record-based curation task +- Create a Grid session for a record-based curation task +- Synchronize the Grid session to pick up schema changes made after the session was created - Download metadata from the Grid to a local CSV - Edit the metadata locally - Upload the metadata back into the Grid @@ -113,7 +114,24 @@ from synapseclient.extensions.curator import get_or_create_curator_grid latest_grid = get_or_create_curator_grid(task_id=curation_task.task_id) ``` -### Step 4: Download record-based metadata as a local CSV +### Step 4: Synchronize the grid session to pick up schema updates + +A Grid session captures the JSON schema in place at the moment it's created — it does not automatically pick up a newer schema version. If the administrator adds or changes a column in the schema *after* you created your session in Step 3, synchronize the session to pull in the latest schema and data from the RecordSet before you continue working. + +For record-based grids, `sync_type` is required — you must explicitly choose `"PULL"` or `"PULL_PUSH"`: + +```python +latest_grid = latest_grid.synchronize(sync_type="PULL") +``` + +- **`"PULL"`** — refreshes the grid session with the latest schema and data from the RecordSet, without writing anything back. Use this to preview an incoming schema or data change (for example, a newly added column) before you've made any edits of your own, or simply to catch the session up to the current RecordSet state. +- **`"PULL_PUSH"`** — does the same pull, then immediately writes the grid session's current data back to the RecordSet as a new version. Use this once you're ready to commit your in-progress edits together with the refreshed schema/data. + +> **Note:** Run this step any time you suspect the schema has changed since you opened the session — for example, if the administrator mentions they've published a new schema version, or if a field you expect to see is missing from your downloaded CSV in Step 5. + +If your session is already current (no schema changes since Step 3), you can skip this step entirely. + +### Step 5: Download record-based metadata as a local CSV Download the current grid contents so you can edit them locally — in pandas, Excel, or any tool that reads CSV. @@ -143,7 +161,7 @@ edited_path = "./grid_edited.csv" df.to_csv(edited_path, index=False) ``` -### Step 5: Import edited record-based metadata to Synapse +### Step 6: Import edited record-based metadata to Synapse `import_csv` upserts rows into the grid based on the `upsert_keys` the administrator configured when setting up the `RecordSet`. Existing rows matching on those keys are updated; new rows are inserted. @@ -320,8 +338,8 @@ File-based tasks follow the same overall flow as record-based tasks (Steps 1–9 **No CSV import.** `import_csv` is not currently supported for file-based grids. Instead, you can either: -- Download the CSV (Step 4) as a local reference, make your edits locally, then copy-paste the values back into the Grid UI -- Make edits directly in the Synapse Grid UI — build the session URL from the grid returned in Step 3: `https://www.synapse.org/Grid:default?sessionId={latest_grid.session_id}` +- Download the CSV (Step 5) as a local reference, make your edits locally, then copy-paste the values back into the Grid UI +- Make edits directly in the Synapse Grid UI — Step 3 prints the session URL (`https://www.synapse.org/Grid:default?sessionId=...`) after creating the session **Use `synchronize()` instead of `export_to_record_set()`.** After editing in the Grid UI, push your changes back to the underlying files: @@ -331,20 +349,9 @@ latest_grid.synchronize() This writes the Grid annotation values back to each file as Synapse annotations. There is no versioned RecordSet — the files themselves are updated in place. -**No per-row export report — but in-session validation still works.** There is no versioned RecordSet, so the export report reviewed in Step 8 (`export_to_record_set()` → `get_detailed_validation_results()`) does not apply. However, the in-session validation from Step 6 works identically for file-based grids: a file-based session created from an `initial_query` still carries a bound JSON schema (`grid_json_schema_id`), so `connect()` + `validate_rows()` returns the same per-row `validation_results`. This is your primary contributor-side check for file-based tasks — run it before `synchronize()`. - -```python -from synapseclient.models.curation import GridQuery, QueryRequest, SelectAll - -with latest_grid.connect() as grid: - query_request = QueryRequest(query=GridQuery(column_selection=[SelectAll()])) - query_result = grid.validate_rows(query_request=query_request) +Note: for file-based grids, `sync_type` is not required and always behaves as `"PULL_PUSH"` — there is no separate preview (`"PULL"`) step. - for row in query_result.rows: - print(f"Row ID: {row.row_id}, Validation Result: {row.validation_results}") -``` - -After you call `synchronize()`, the administrator also verifies schema compliance on their end. If they report violations, correct the flagged annotations in the Grid UI and re-synchronize. +**No per-row export report — but in-session validation still works.** There is no versioned RecordSet, so the export report reviewed in Step 8 (`export_to_record_set()` → `get_detailed_validation_results()`) does not apply. However, the in-session validation from Step 6 works identically for file-based grids: a file-based session created from an `initial_query` still carries a bound JSON schema (`grid_json_schema_id`), so `connect()` + `validate_rows()` returns the same per-row `validation_results`. This is your primary contributor-side check for file-based tasks — run it before `synchronize()`. ## Appendix @@ -378,7 +385,7 @@ Deleting is permanent — you can no longer re-export from this session. If you - [Grid.download_csv][synapseclient.models.Grid.download_csv] - Download Grid contents as a local CSV - [Grid.import_csv][synapseclient.models.Grid.import_csv] - Upsert CSV edits back into a Grid session (record-based grids only) - [Grid.export_to_record_set][synapseclient.models.Grid.export_to_record_set] - Export Grid data back to RecordSet and generate validation results -- [Grid.synchronize][synapseclient.models.Grid.synchronize] - Synchronize a file-based Grid against its source file view +- [Grid.synchronize][synapseclient.models.Grid.synchronize] - Synchronize a Grid session against its source RecordSet or file view, pulling in schema/data changes and (for `PULL_PUSH`) writing edits back - [Grid.delete][synapseclient.models.Grid.delete] - Delete a Grid session - [RecordSet.get_detailed_validation_results][synapseclient.models.RecordSet.get_detailed_validation_results] - Retrieve the row-level validation report for a RecordSet diff --git a/docs/reference/experimental/async/curator.md b/docs/reference/experimental/async/curator.md index 142f1eba9..e0dccc9e2 100644 --- a/docs/reference/experimental/async/curator.md +++ b/docs/reference/experimental/async/curator.md @@ -16,6 +16,7 @@ - list_async - create_grid_session_async - set_task_state_async + - synchronize_active_grid_session_async --- [](){ #RecordSet-reference-async } @@ -56,6 +57,12 @@ inherited_members: true members: --- +[](){ #SyncType-reference-async } +::: synapseclient.models.SyncType + options: + inherited_members: true + members: +--- [](){ #grid-reference-async } ::: synapseclient.models.Grid options: diff --git a/docs/reference/experimental/sync/curator.md b/docs/reference/experimental/sync/curator.md index 71a3413fe..7d872fa9a 100644 --- a/docs/reference/experimental/sync/curator.md +++ b/docs/reference/experimental/sync/curator.md @@ -16,6 +16,7 @@ - list - create_grid_session - set_task_state + - synchronize_active_grid_session --- [](){ #RecordSet-reference } @@ -56,6 +57,12 @@ inherited_members: true members: --- +[](){ #SyncType-reference } +::: synapseclient.models.SyncType + options: + inherited_members: true + members: +--- [](){ #grid-reference } ::: synapseclient.models.Grid options: diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index 90e08c0ad..68a0164af 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -15,6 +15,7 @@ Grid, GridExecutionDetails, RecordBasedMetadataTaskProperties, + SyncType, TaskExecutionDetails, TaskState, ) @@ -126,6 +127,7 @@ "TaskState", "Grid", "GridExecutionDetails", + "SyncType", "TaskExecutionDetails", "UserProfile", "UserPreference", diff --git a/synapseclient/models/curation.py b/synapseclient/models/curation.py index 81841b6e7..11863efbe 100644 --- a/synapseclient/models/curation.py +++ b/synapseclient/models/curation.py @@ -79,7 +79,10 @@ merge_dataclass_entities, ) from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator -from synapseclient.models.mixins.enum_coercion import EnumCoercionMixin +from synapseclient.models.mixins.enum_coercion import ( + EnumCoercionMixin, + ForwardCompatibleStrEnum, +) from synapseclient.models.recordset import ValidationSummary from synapseclient.models.table_components import Column, CsvTableDescriptor, Query @@ -131,6 +134,24 @@ class AuthorizationMode(str, Enum): ownership team. User visibility of rows depends on their individual permissions.""" +class SyncType(ForwardCompatibleStrEnum): + """ + The type of synchronization to perform on a grid session. + + See . + """ + + PULL = "PULL" + """Update the grid with the latest data from the source, without writing the + grid back to the source. Currently only supported for RecordSet-based grids.""" + + PULL_PUSH = "PULL_PUSH" + """Update the grid with the latest data from the source, then update the source + (the referenced entities for EntityView-based grids, or the source RecordSet for + RecordSet-based grids) with the grid data. This is the default when sync_type is + not specified.""" + + @dataclass class FileBasedMetadataTaskProperties(EnumCoercionMixin): """ @@ -812,6 +833,81 @@ def create_grid_session( """ return Grid() + def synchronize_active_grid_session( + self, + *, + sync_type: Optional["SyncType | str"] = None, + synapse_client: Synapse | None = None, + ) -> "Grid": + """ + Synchronize this task's active grid session against its source entity, + creating a new grid session first if the task does not already have one. + + FileBasedMetadataTaskProperties tasks always perform a SyncType.PULL_PUSH; + any sync_type passed in is ignored for those tasks. RecordBasedMetadataTaskProperties + tasks require sync_type to be provided explicitly. + + Arguments: + sync_type: The type of synchronization to perform: + + - SyncType.PULL: Update the grid session with the latest data/schema + from the source RecordSet, without writing the grid back to it. + Use this to preview an incoming schema or data change in the grid + before committing it. Only supported for record-based tasks. + - SyncType.PULL_PUSH: Update the grid session with the latest data + from the source, then write the grid's data back to the source + (the source RecordSet for record-based tasks, or the referenced + entities for file-based tasks). This commits any in-progress + curation in the grid as a new version of the source. + + Required for record-based tasks (pass PULL to preview or PULL_PUSH + to commit). Ignored for file-based tasks, which always use + SyncType.PULL_PUSH. + synapse_client: If not passed in and caching was not disabled by + Synapse.allow_client_caching(False) this will use the last created + instance from the Synapse class constructor. + + Returns: + The synchronized Grid. + + Raises: + ValueError: If task_id is unset, task_properties is of an unsupported + type, or sync_type is not provided for a record-based task. + + Example: Synchronize a record-based curation task's grid session +   + + ```python + from synapseclient import Synapse + from synapseclient.models import CurationTask + from synapseclient.models.curation import SyncType + + syn = Synapse() + syn.login() + + grid = CurationTask(task_id=123).synchronize_active_grid_session( + sync_type=SyncType.PULL_PUSH + ) + ``` + + Example: Synchronize a file-based curation task's grid session +   + + File-based tasks always synchronize with SyncType.PULL_PUSH, so + sync_type can be omitted entirely. + + ```python + from synapseclient import Synapse + from synapseclient.models import CurationTask + + syn = Synapse() + syn.login() + + grid = CurationTask(task_id=456).synchronize_active_grid_session() + ``` + """ + return Grid() + def delete( self, delete_source: bool = False, @@ -2097,6 +2193,145 @@ async def main(): task = cls().fill_from_dict(synapse_response=task_dict) yield task + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: ( + f"CurationTask_SynchronizeActiveGridSession: ID: {self.task_id}" + ) + ) + async def synchronize_active_grid_session_async( + self, + *, + sync_type: Optional[Union["SyncType", str]] = None, + synapse_client: Optional[Synapse] = None, + ) -> Optional["Grid"]: + """ + Synchronize this task's active grid session against its source entity. + + If task_properties is not yet populated on this object, it is fetched + from Synapse first. If the task has no active grid session, a warning + is logged and None is returned; no new grid session is created. + + FileBasedMetadataTaskProperties tasks always perform a SyncType.PULL_PUSH; + any sync_type passed in is ignored for those + tasks. RecordBasedMetadataTaskProperties tasks require sync_type to be + provided explicitly. + + Arguments: + sync_type: The type of synchronization to perform: + + - SyncType.PULL: Update the grid session with the latest data/schema + from the source RecordSet, without writing the grid back to it. + Use this to preview an incoming schema or data change in the grid + before committing it. Only supported for record-based tasks. + - SyncType.PULL_PUSH: Update the grid session with the latest data + from the source, then write the grid's data back to the source + (the source RecordSet for record-based tasks, or the referenced + entities for file-based tasks). This commits any in-progress + curation in the grid as a new version of the source. + + Required for record-based tasks (pass PULL to preview or PULL_PUSH + to commit). Ignored for file-based tasks, which always use + SyncType.PULL_PUSH. + synapse_client: If not passed in and caching was not disabled by + Synapse.allow_client_caching(False) this will use the last created + instance from the Synapse class constructor. + + Returns: + The synchronized Grid, or None if the task has no active grid session. + + Raises: + ValueError: If task_id is unset, task_properties is of an unsupported + type, or sync_type is not provided for a record-based task. + + Example: Synchronize a record-based curation task's grid session +   + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import CurationTask + from synapseclient.models.curation import SyncType + + syn = Synapse() + syn.login() + + async def main(): + grid = await CurationTask(task_id=123).synchronize_active_grid_session_async( + sync_type=SyncType.PULL_PUSH + ) + if grid is not None: + print(grid.session_id) + + asyncio.run(main()) + ``` + + Example: Synchronize a file-based curation task's grid session +   + + File-based tasks always synchronize with SyncType.PULL_PUSH, so + sync_type can be omitted entirely. + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import CurationTask + + syn = Synapse() + syn.login() + + async def main(): + grid = await CurationTask(task_id=456).synchronize_active_grid_session_async() + if grid is not None: + print(grid.session_id) + + asyncio.run(main()) + ``` + """ + client = Synapse.get_client(synapse_client=synapse_client) + + if not self.task_properties: + await self.get_async(synapse_client=synapse_client) + + if isinstance(self.task_properties, FileBasedMetadataTaskProperties): + if sync_type is not None and sync_type != SyncType.PULL_PUSH: + client.logger.warning( + f"Ignoring sync_type={sync_type} for CurationTask " + f"{self.task_id}: FileBasedMetadataTaskProperties tasks always " + "use SyncType.PULL_PUSH." + ) + sync_type = SyncType.PULL_PUSH + elif isinstance(self.task_properties, RecordBasedMetadataTaskProperties): + if not sync_type: + raise ValueError( + "sync_type must be provided for RecordBasedMetadataTaskProperties" + ) + else: + raise ValueError( + f"Synchronization only supports FileBasedMetadataTaskProperties or " + f"RecordBasedMetadataTaskProperties, got {type(self.task_properties).__name__}." + ) + + status = await self.get_status_async(synapse_client=synapse_client) + if ( + status.execution_details is None + or status.execution_details.active_session_id is None + ): + client.logger.warning( + f"No active grid session found for task {self.task_id}. Skipping " + "synchronization." + ) + return None + active_grid_session_id = status.execution_details.active_session_id + + client.logger.info( + f"Synchronizing active grid session {active_grid_session_id} for " + f"task {self.task_id}" + ) + grid = Grid(session_id=active_grid_session_id) + return await grid.synchronize_async( + synapse_client=synapse_client, sync_type=sync_type + ) + @dataclass class CreateGridRequest(EnumCoercionMixin, AsynchronousCommunicator): @@ -3288,7 +3523,7 @@ def to_synapse_request(self) -> Dict[str, Any]: @dataclass -class SynchronizeGridRequest(AsynchronousCommunicator): +class SynchronizeGridRequest(EnumCoercionMixin, AsynchronousCommunicator): """ A request to synchronize a grid session. @@ -3300,12 +3535,19 @@ class SynchronizeGridRequest(AsynchronousCommunicator): grid_session_id: str """The ID of the grid session to synchronize.""" + sync_type: Optional[Union[SyncType, str]] = field(default=None) + """The type of synchronization to perform. Optional; the server defaults to + SyncType.PULL_PUSH when omitted. SyncType.PULL is currently only supported for + RecordSet-based grids.""" + concrete_type: str = field(default=SYNCHRONIZE_GRID_REQUEST) """The concrete type for this request.""" error_messages: Optional[list[str]] = field(default=None, compare=False) """Any error messages generated during the synchronization process.""" + _ENUM_FIELDS: ClassVar[dict[str, type]] = {"sync_type": SyncType} + def fill_from_dict( self, synapse_response: Dict[str, Any] ) -> "SynchronizeGridRequest": @@ -3328,10 +3570,13 @@ def to_synapse_request(self) -> Dict[str, Any]: Returns: A dictionary representation of this object for API requests. """ - return { + request_dict = { "concreteType": self.concrete_type, "gridSessionId": self.grid_session_id, + "syncType": self.sync_type.value if self.sync_type is not None else None, } + delete_none_keys(request_dict) + return request_dict @dataclass @@ -3703,15 +3948,27 @@ def export_to_record_set( return self def synchronize( - self, *, timeout: int = 120, synapse_client: Optional[Synapse] = None + self, + *, + sync_type: Optional[Union[SyncType, str]] = None, + timeout: int = 120, + synapse_client: Optional[Synapse] = None, ) -> "Grid": """ Synchronizes the grid session's schema and row data against its source entity. - This is intended for grid sessions created from a file view via `initial_query`. - Grid sessions backed by a RecordSet should use `export_to_record_set` instead. + Grid sessions created from a file view via `initial_query` always perform a + full PULL_PUSH. Grid sessions backed by a RecordSet may instead pass + `sync_type=SyncType.PULL` to pull the latest RecordSet data/schema into the + session for review, without immediately writing the merged result back as a + new RecordSet version. Once satisfied with the result, call this method again + (with `sync_type` omitted, or explicitly set to `SyncType.PULL_PUSH`) to push + the merged data back to the RecordSet as a new version. Arguments: + sync_type: The type of synchronization to perform. Optional; the server + defaults to `SyncType.PULL_PUSH` when omitted. `SyncType.PULL` is + currently only supported for RecordSet-based grids. timeout: The number of seconds to wait for the job to complete or progress before raising a SynapseTimeoutError. Defaults to 120. synapse_client: If not passed in and caching was not disabled by @@ -3743,6 +4000,29 @@ def synchronize( # Synchronize the grid with the latest state of the file view grid = grid.synchronize() ``` + + Example: Preview a RecordSet-backed grid's merge before pushing it back +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Grid + from synapseclient.models.curation import SyncType + + syn = Synapse() + syn.login() + + grid = Grid(record_set_id="syn1234567") + grid = grid.create() + + # Pull in the latest RecordSet data/schema without pushing back yet + grid = grid.synchronize(sync_type=SyncType.PULL) + + # ... review the merged result in the grid session ... + + # Push the merged result back as a new RecordSet version + grid = grid.synchronize(sync_type=SyncType.PULL_PUSH) + ``` """ return self @@ -4849,15 +5129,27 @@ async def main(): method_to_trace_name=lambda self, **kwargs: f"Grid_Synchronize: ID: {self.session_id}" ) async def synchronize_async( - self, *, timeout: int = 120, synapse_client: Optional[Synapse] = None + self, + *, + sync_type: Optional[Union[SyncType, str]] = None, + timeout: int = 120, + synapse_client: Optional[Synapse] = None, ) -> "Grid": """ Synchronizes the grid session's schema and row data against its source entity. - This is intended for grid sessions created from a file view via `initial_query`. - Grid sessions backed by a RecordSet should use `export_to_record_set` instead. + Grid sessions created from a file view via `initial_query` always perform a + full PULL_PUSH. Grid sessions backed by a RecordSet may instead pass + `sync_type=SyncType.PULL` to pull the latest RecordSet data/schema into the + session for review, without immediately writing the merged result back as a + new RecordSet version. Once satisfied with the result, call this method again + (with `sync_type` omitted, or explicitly set to `SyncType.PULL_PUSH`) to push + the merged data back to the RecordSet as a new version. Arguments: + sync_type: The type of synchronization to perform. Optional; the server + defaults to `SyncType.PULL_PUSH` when omitted. `SyncType.PULL` is + currently only supported for RecordSet-based grids. timeout: The number of seconds to wait for the job to complete or progress before raising a SynapseTimeoutError. Defaults to 120. synapse_client: If not passed in and caching was not disabled by @@ -4893,11 +5185,40 @@ async def main(): asyncio.run(main()) ``` + + Example: Preview a RecordSet-backed grid's merge before pushing it back +   + + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Grid + from synapseclient.models.curation import SyncType + + syn = Synapse() + syn.login() + + async def main(): + grid = Grid(record_set_id="syn1234567") + grid = await grid.create_async() + + # Pull in the latest RecordSet data/schema without pushing back yet + grid = await grid.synchronize_async(sync_type=SyncType.PULL) + + # ... review the merged result in the grid session ... + + # Push the merged result back as a new RecordSet version + grid = await grid.synchronize_async(sync_type=SyncType.PULL_PUSH) + + asyncio.run(main()) + ``` """ if not self.session_id: raise ValueError("session_id is required to synchronize a GridSession") - request = SynchronizeGridRequest(grid_session_id=self.session_id) + request = SynchronizeGridRequest( + grid_session_id=self.session_id, sync_type=sync_type + ) result = await request.send_job_and_wait_async( timeout=timeout, synapse_client=synapse_client ) diff --git a/tests/integration/synapseclient/models/async/test_grid_async.py b/tests/integration/synapseclient/models/async/test_grid_async.py index 502b86ce8..1a0480b97 100644 --- a/tests/integration/synapseclient/models/async/test_grid_async.py +++ b/tests/integration/synapseclient/models/async/test_grid_async.py @@ -284,7 +284,7 @@ async def test_delete_grid_session_validation_error_async(self) -> None: ): await grid.delete_async(synapse_client=self.syn) - async def test_synchronize_grid_async( + async def test_synchronize_grid_entity_view_async( self, entity_view: tuple[Folder, EntityView], ) -> None: @@ -320,7 +320,9 @@ async def file_indexed() -> bool: ) # WHEN: Synchronizing the same session - synced_grid = await created_grid.synchronize_async(synapse_client=self.syn) + synced_grid = await created_grid.synchronize_async( + synapse_client=self.syn, sync_type="PULL_PUSH" + ) # THEN: The session ID is unchanged assert synced_grid.session_id == created_grid.session_id @@ -337,6 +339,26 @@ async def file_indexed() -> bool: df = pd.read_csv(csv_path) assert uploaded_file.id in df["id"].tolist() + async def test_synchronize_grid_recordset_async( + self, + record_set_fixture: RecordSet, + ) -> None: + # GIVEN: A Grid session created at T0 from a RecordSet + grid = Grid(record_set_id=record_set_fixture.id) + created_grid = await grid.create_async( + timeout=ASYNC_JOB_TIMEOUT_SEC, synapse_client=self.syn + ) + self.schedule_for_cleanup(created_grid) + + # WHEN: Synchronizing the same session + synced_grid = await created_grid.synchronize_async( + synapse_client=self.syn, sync_type="PULL_PUSH" + ) + + # THEN: The session ID is unchanged and the source entity is still the RecordSet + assert synced_grid.session_id == created_grid.session_id + assert synced_grid.source_entity_id == record_set_fixture.id + async def test_import_csv_to_grid_session_async( self, record_set_fixture: RecordSet, diff --git a/tests/unit/synapseclient/models/async/unit_test_curation_async.py b/tests/unit/synapseclient/models/async/unit_test_curation_async.py index 09d78f92b..bdedb8f72 100644 --- a/tests/unit/synapseclient/models/async/unit_test_curation_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_curation_async.py @@ -56,6 +56,7 @@ SelectColumn, SelectSelection, SynchronizeGridRequest, + SyncType, TaskState, UploadToTablePreviewRequest, _create_task_properties_from_dict, @@ -1393,6 +1394,318 @@ async def test_list_async_state_filter_invalid_string_raises(self) -> None: pass # pragma: no cover +class TestCurationTaskSynchronizeActiveGridSession: + """Unit tests for CurationTask.synchronize_active_grid_session_async.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + async def test_record_based_creates_session_when_none_active(self) -> None: + """When there is no active session, a new one is created and its session_id is used to synchronize.""" + # GIVEN a record-based task with no active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response(), + ) as mock_get_status, + patch.object( + task, + "create_grid_session_async", + new_callable=AsyncMock, + ) as mock_create_grid_session, + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + created_grid = MagicMock() + created_grid.session_id = SESSION_ID + mock_create_grid_session.return_value = created_grid + + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I synchronize the active grid session + result = await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL, synapse_client=self.syn + ) + + # THEN no session was created and nothing is returned + assert result is None + + async def test_record_based_reuses_existing_session(self) -> None: + """When a session is already active, it is reused and no new session is created.""" + # GIVEN a record-based task with an already-active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch.object( + task, + "create_grid_session_async", + new_callable=AsyncMock, + ) as mock_create_grid_session, + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I synchronize the active grid session + result = await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL_PUSH, synapse_client=self.syn + ) + + # THEN no new grid session is created, and the existing session is synchronized + mock_create_grid_session.assert_not_called() + mock_grid_cls.assert_called_once_with(session_id=SESSION_ID) + mock_grid.synchronize_async.assert_called_once_with( + synapse_client=self.syn, sync_type=SyncType.PULL_PUSH + ) + assert result is mock_grid + + async def test_record_based_without_sync_type_raises(self) -> None: + """Record-based tasks require an explicit sync_type.""" + # GIVEN a record-based task + task = CurationTask( + task_id=TASK_ID, + task_properties=RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ), + ) + + # WHEN I call synchronize_active_grid_session_async without a sync_type + # THEN it should raise ValueError + with pytest.raises( + ValueError, + match="sync_type must be provided for RecordBasedMetadataTaskProperties", + ): + await task.synchronize_active_grid_session_async(synapse_client=self.syn) + + async def test_unrecognized_sync_type_string_is_forward_compatible(self) -> None: + """SyncType is forward-compatible, so an unrecognized string is passed + through rather than rejected, in case the server has added a new value.""" + # GIVEN a record-based task with an already-active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I call synchronize_active_grid_session_async with an unrecognized + # string THEN no exception is raised, and the value is forwarded as-is + await task.synchronize_active_grid_session_async( + sync_type="INVALID", synapse_client=self.syn + ) + + mock_grid.synchronize_async.assert_called_once_with( + synapse_client=self.syn, sync_type="INVALID" + ) + + async def test_valid_sync_type_string_is_coerced(self) -> None: + """Valid sync_type strings are coerced to SyncType enum.""" + # GIVEN a record-based task with an already-active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I pass sync_type as a string "PULL_PUSH" + await task.synchronize_active_grid_session_async( + sync_type="PULL_PUSH", synapse_client=self.syn + ) + + # THEN the string is coerced to the enum and used correctly + mock_grid.synchronize_async.assert_called_once_with( + synapse_client=self.syn, sync_type=SyncType.PULL_PUSH + ) + + async def test_file_based_ignores_sync_type(self) -> None: + """File-based tasks always synchronize with PULL_PUSH regardless of the sync_type passed in.""" + # GIVEN a file-based task with an already-active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=FileBasedMetadataTaskProperties( + upload_folder_id=UPLOAD_FOLDER_ID, file_view_id=FILE_VIEW_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch.object( + task, "create_grid_session_async", new_callable=AsyncMock + ) as mock_create_grid_session, + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I pass sync_type=PULL (only valid for record-based tasks) + await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL, synapse_client=self.syn + ) + + # THEN the file-based task ignores it and always synchronizes with PULL_PUSH + mock_create_grid_session.assert_not_called() + mock_grid.synchronize_async.assert_called_once_with( + synapse_client=self.syn, sync_type=SyncType.PULL_PUSH + ) + + async def test_file_based_without_sync_type(self) -> None: + """File-based tasks do not require sync_type to be provided.""" + # GIVEN a file-based task with an already-active grid session + task = CurationTask( + task_id=TASK_ID, + task_properties=FileBasedMetadataTaskProperties( + upload_folder_id=UPLOAD_FOLDER_ID, file_view_id=FILE_VIEW_ID + ), + ) + + with ( + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I synchronize without providing sync_type + await task.synchronize_active_grid_session_async(synapse_client=self.syn) + + # THEN it defaults to PULL_PUSH without raising + mock_grid.synchronize_async.assert_called_once_with( + synapse_client=self.syn, sync_type=SyncType.PULL_PUSH + ) + + async def test_fetches_task_properties_when_missing(self) -> None: + """If task_properties is not yet populated, it is fetched from Synapse first.""" + # GIVEN a CurationTask with only a task_id set (no task_properties) + task = CurationTask(task_id=TASK_ID) + + async def fake_get_async(*, synapse_client=None): + task.task_properties = RecordBasedMetadataTaskProperties( + record_set_id=RECORD_SET_ID + ) + return task + + with ( + patch.object( + task, "get_async", new_callable=AsyncMock, side_effect=fake_get_async + ) as mock_get_async, + patch( + "synapseclient.models.curation.get_curation_task_status", + new_callable=AsyncMock, + return_value=_get_curation_task_status_response( + active_session_id=SESSION_ID + ), + ), + patch("synapseclient.models.curation.Grid") as mock_grid_cls, + ): + mock_grid = mock_grid_cls.return_value + mock_grid.synchronize_async = AsyncMock(return_value=mock_grid) + + # WHEN I synchronize the active grid session + await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL_PUSH, synapse_client=self.syn + ) + + # THEN task_properties was fetched before the type check ran + mock_get_async.assert_called_once_with(synapse_client=self.syn) + assert isinstance(task.task_properties, RecordBasedMetadataTaskProperties) + + async def test_without_task_id_raises(self) -> None: + """Without a task_id, fetching task_properties fails with ValueError.""" + # GIVEN a CurationTask with neither task_id nor task_properties set + task = CurationTask() + + # WHEN I call synchronize_active_grid_session_async + # THEN it should raise ValueError (propagated from get_async) + with pytest.raises(ValueError, match="task_id is required to get"): + await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL_PUSH, synapse_client=self.syn + ) + + async def test_unsupported_task_properties_type_raises(self) -> None: + """Unsupported task_properties types raise ValueError with the type name.""" + + # GIVEN a fake task properties type that doesn't exist yet + class ComputeBasedMetadataTaskProperties: + """A hypothetical future task properties type.""" + + pass + + # AND a CurationTask with this unsupported type + task = CurationTask(task_id=TASK_ID) + task.task_properties = ComputeBasedMetadataTaskProperties() + + # WHEN I call synchronize_active_grid_session_async + # THEN it should raise ValueError mentioning the actual type name + with pytest.raises( + ValueError, + match="Synchronization only supports FileBasedMetadataTaskProperties or " + "RecordBasedMetadataTaskProperties, got ComputeBasedMetadataTaskProperties", + ): + await task.synchronize_active_grid_session_async( + sync_type=SyncType.PULL_PUSH, synapse_client=self.syn + ) + + class TestGrid: """Unit tests for the Grid model.""" @@ -2393,10 +2706,16 @@ async def test_download_csv_async_empty_file_handle_id(self): class TestSynchronizeGridRequest: - def test_to_synapse_request(self) -> None: - # GIVEN a SynchronizeGridRequest with all fields set + + @pytest.mark.parametrize( + "sync_type", + [None, SyncType.PULL, SyncType.PULL_PUSH, "PULL", "PULL_PUSH"], + ids=["omitted", "pull", "pull_push", "string_pull", "string_pull_push"], + ) + def test_to_synapse_request(self, sync_type: SyncType) -> None: + # GIVEN a SynchronizeGridRequest with the given sync_type sync_req = SynchronizeGridRequest( - grid_session_id=SESSION_ID, + grid_session_id=SESSION_ID, sync_type=sync_type ) # WHEN I convert it to a synapse request @@ -2406,6 +2725,27 @@ def test_to_synapse_request(self) -> None: assert "concreteType" in result assert result["gridSessionId"] == SESSION_ID + # AND syncType is omitted when not set, and EnumCoercionMixin normalizes it to a SyncType + # member on assignment + if sync_type is None: + assert "syncType" not in result + else: + assert result["syncType"] == SyncType(sync_type).value + + def test_unrecognized_sync_type_is_forward_compatible(self) -> None: + # GIVEN a sync_type that doesn't match any declared SyncType member + # (wrong case, or a value the server may add in the future) + # WHEN constructing a SynchronizeGridRequest with it + # THEN SyncType is forward-compatible, so it is accepted as-is + # rather than rejected, in case the server has added a new value + sync_req = SynchronizeGridRequest(grid_session_id=SESSION_ID, sync_type="pull") + assert sync_req.sync_type == "pull" + + sync_req = SynchronizeGridRequest( + grid_session_id=SESSION_ID, sync_type="NOT_REAL" + ) + assert sync_req.sync_type == "NOT_REAL" + def test_fill_from_dict(self) -> None: # GIVEN a response with synchronize grid session data raw_response = {