[SYNPY-1840] Add ability to set column order when creating file and record based tasks.

docs/guides/extensions/curator/metadata_curation.md

@@ -132,6 +132,55 @@ print(f"Created CurationTask: {task.task_id}")
 - Automatic schema binding to the folder for validation
 - Optional wiki attached to the folder
 
+### Controlling the order of the columns
+
+Both [create_record_based_metadata_task][synapseclient.extensions.curator.create_record_based_metadata_task] and [create_file_based_metadata_task][synapseclient.extensions.curator.create_file_based_metadata_task] accept an optional `column_order` that controls the left-to-right order of the columns contributors see in the grid. JSON Schema property order is not reliably preserved by downstream applications, so pass the order explicitly here instead of relying on the schema.
+
+You only need to name the columns that need intentional placement. Every column you leave out stays visible and is appended after the ones you named, in its existing relative order. This keeps the configuration short as properties are added to the schema over time.
+
+A few columns are always pinned to the front and cannot be moved:
+
+- Record-based tasks pin the `upsert_keys`, because they identify each row.
+- File-based tasks pin `name` and `id`.
+
+Naming a pinned column in `column_order` has no effect — it stays in its pinned position and is not duplicated. A name that does not match any available column raises a `ValueError`.
+
+```python
+record_set, curation_task = create_record_based_metadata_task(
+    synapse_client=syn,
+    folder_id="syn987654321",
+    record_set_name="AnimalMetadata_Records",
+    record_set_description="Centralized metadata for animal study data",
+    curation_task_name="AnimalMetadata_Curation",
+    upsert_keys=["StudyKey"],
+    column_order=["diagnosis", "specimenType", "assay"],
+    instructions="Complete all required fields according to the schema.",
+    schema_uri=schema_uri,
+    create_grid=False,
+)
+
+# Resulting column order:
+# StudyKey, diagnosis, specimenType, assay, <remaining schema properties>
+```
+
+```python
+entity_view, task = create_file_based_metadata_task(
+    synapse_client=syn,
+    folder_id="syn987654321",
+    curation_task_name="FileMetadata_Curation",
+    instructions="Annotate each file with metadata according to the schema requirements.",
+    entity_view_name="Animal Study Files View",
+    schema_uri=schema_uri,
+    column_order=["patientId", "sampleId", "assay", "fileFormat"],
+    return_entities=True,
+)
+
+# Resulting column order:
+# name, id, patientId, sampleId, assay, fileFormat, <remaining columns>
+```
+
+For file-based tasks the available columns include the Synapse managed columns such as `createdBy` and `modifiedOn` in addition to the JSON Schema properties, so you may place those wherever you like. They are no longer pinned to the front — only `name` and `id` are — which lets contributor-relevant metadata appear before system metadata.
+
 ### Controlling who can access the grid session
 
 Both [create_record_based_metadata_task][synapseclient.extensions.curator.create_record_based_metadata_task] and [create_file_based_metadata_task][synapseclient.extensions.curator.create_file_based_metadata_task] accept an optional `authorization_mode` that tells clients how to scope access when a grid session is created for the task:

synapseclient/extensions/curator/file_based_metadata_task.py

@@ -5,12 +5,17 @@
 in Synapse, including EntityView creation, CurationTask setup, and Wiki attachment.
 """
 
+from collections import OrderedDict
 from typing import Any, Optional, Tuple, Union
 
 from synapseclient import Synapse  # type: ignore
 from synapseclient import Wiki  # type: ignore
 from synapseclient.core.exceptions import SynapseHTTPError  # type: ignore
-from synapseclient.extensions.curator.utils import project_id_from_entity_id
+from synapseclient.extensions.curator.utils import (
+    project_id_from_entity_id,
+    resolve_column_order_list,
+    validate_column_order_list,
+)
 from synapseclient.models import (  # type: ignore
     AuthorizationMode,
     Column,
@@ -42,6 +47,7 @@ def _create_json_schema_entity_view(
     synapse_entity_id: str,
     entity_view_name: str = "JSON Schema view",
     view_type_mask: Union[int, ViewTypeMask] = ViewTypeMask.FILE,
+    column_order: list[str] | None = None,
     syn: Optional[Synapse] = None,
 ) -> EntityView:
     """
@@ -55,17 +61,29 @@ def _create_json_schema_entity_view(
             ViewTypeMask.FILE. Additional types can be added using bitwise OR
             (e.g., ViewTypeMask.FILE | ViewTypeMask.DOCKER). Accepts either a
             ViewTypeMask enum member or its raw integer value.
+        column_order: Optional list of column names to place immediately after the
+            pinned name and id columns, in the order given. Remaining columns keep
+            their existing relative order.
         syn: A Synapse object thats been logged in
 
     Returns:
         The created EntityView object
+
+    Raises:
+        ValueError: If synapse_entity_id is not a Folder or a Project, or if
+            column_order is malformed or names a column that is not present on the
+            created EntityView.
     """
     entity = get(
         file_options=FileOptions(download_file=False),
         synapse_id=synapse_entity_id,
         synapse_client=syn,
     )
-    assert isinstance(entity, (Folder, Project))
+    if not isinstance(entity, (Folder, Project)):
+        raise ValueError(
+            f"A JSON Schema can only be read from a Folder or a Project, but "
+            f"{synapse_entity_id} is a {type(entity).__name__}."
+        )
     jsb = entity.get_schema(synapse_client=syn)
     version_info = jsb.json_schema_version_info
     schema = JSONSchema(version_info.schema_name, version_info.organization_name)
@@ -78,10 +96,27 @@ def _create_json_schema_entity_view(
         view_type_mask=view_type_mask,
         columns=columns,
     ).store(synapse_client=syn)
-    # This reorder is so that these show up in the front of the EntityView in Synapse.
-    view.reorder_column(name="name", index=0)
-    view.reorder_column(name="id", index=1)
-    view.reorder_column(name="createdBy", index=2)
+
+    try:
+        available_columns = list(view.columns.keys())
+        ordered_columns = resolve_column_order_list(
+            available_columns=available_columns,
+            pinned_columns=["name", "id"],
+            requested_columns=column_order,
+        )
+        view.columns = OrderedDict(
+            (column, view.columns[column]) for column in ordered_columns
+        )
+    except ValueError:
+        try:
+            view.delete(synapse_client=syn)
+        except Exception:
+            Synapse.get_client(synapse_client=syn).logger.exception(
+                f"Could not delete the created EntityView {view.id}. Delete it "
+                "yourself, either from the Synapse web UI, or with the Python "
+                f"client: EntityView(id='{view.id}').delete()"
+            )
+        raise
     view.store(synapse_client=syn)
     return view
 
@@ -332,6 +367,7 @@ def create_file_based_metadata_task(
     # parameter and change the return type to Tuple[EntityView, CurationTask].
     return_entities: bool = False,
     *,
+    column_order: list[str] | None = None,
     synapse_client: Optional[Synapse] = None,
 ) -> Union[Tuple[str, int], Tuple[EntityView, CurationTask]]:
     """
@@ -385,6 +421,32 @@ def create_file_based_metadata_task(
         )
         ```
 
+    Example: Controlling the column order of the EntityView
+        Pass column_order to place specific columns immediately after the pinned name
+        and id columns. You only need to name the columns you care about; every other
+        column, including Synapse managed columns such as createdBy, is appended
+        afterwards in its existing order.
+
+        ```python
+        import synapseclient
+        from synapseclient.extensions.curator import create_file_based_metadata_task
+
+        syn = synapseclient.Synapse()
+        syn.login()
+
+        entity_view, curation_task = create_file_based_metadata_task(
+            synapse_client=syn,
+            folder_id="syn12345678",
+            curation_task_name="BiospecimenMetadataTemplate",
+            instructions="Please curate this metadata according to the schema requirements",
+            column_order=["patientId", "sampleId", "assay", "fileFormat"],
+            return_entities=True,
+        )
+
+        # Resulting column order:
+        # name, id, patientId, sampleId, assay, fileFormat, <remaining columns>
+        ```
+
     Arguments:
         folder_id: The Synapse Folder ID to create the file view for.
         curation_task_name: Name for the CurationTask (used as data_type field).
@@ -423,6 +485,14 @@ def create_file_based_metadata_task(
             objects instead of their ID strings. Defaults to False for backwards
             compatibility. The entity-returning shape will become the default in
             v5.0.0, at which point this parameter will be removed.
+        column_order: Optional list of column names placed immediately after the
+            pinned name and id columns, in the order given. Columns that are not
+            named keep their existing relative order and are appended afterwards, so
+            you only need to list the columns that need intentional placement. The
+            name and id columns always remain the two leftmost columns, so naming
+            either of them here has no effect. Every name must match a column on the
+            created EntityView, which includes the JSON Schema properties as well as
+            the Synapse managed columns such as createdBy and modifiedOn.
         synapse_client: If not passed in and caching was not disabled by
                 `Synapse.allow_client_caching(False)` this will use the last created
                 instance from the Synapse class constructor.
@@ -437,7 +507,8 @@ def create_file_based_metadata_task(
           - The created CurationTask object
 
     Raises:
-        ValueError: If required parameters are missing.
+        ValueError: If required parameters are missing, or if column_order is
+            malformed or names a column that is not on the created EntityView.
         SynapseError: If there are issues with Synapse operations.
     """
     # Validate required parameters
@@ -447,6 +518,7 @@ def create_file_based_metadata_task(
         raise ValueError("curation_task_name is required")
     if not instructions:
         raise ValueError("instructions is required")
+    validate_column_order_list(column_order)
 
     synapse_client = Synapse.get_client(synapse_client=synapse_client)
 
@@ -478,6 +550,7 @@ def create_file_based_metadata_task(
             synapse_entity_id=folder_id,
             entity_view_name=entity_view_name,
             view_type_mask=view_type_mask,
+            column_order=column_order,
         )
         entity_view_id = entity_view.id
     except Exception as e:

synapseclient/extensions/curator/record_based_metadata_task.py

@@ -12,7 +12,11 @@
 from synapseclient import Synapse
 from synapseclient.core.typing_utils import DataFrame as DATA_FRAME_TYPE
 from synapseclient.core.utils import test_import_pandas
-from synapseclient.extensions.curator.utils import project_id_from_entity_id
+from synapseclient.extensions.curator.utils import (
+    project_id_from_entity_id,
+    resolve_column_order_list,
+    validate_column_order_list,
+)
 from synapseclient.models import (
     AuthorizationMode,
     CurationTask,
@@ -78,33 +82,6 @@ def extract_schema_properties_from_dict(schema_data: Dict[str, Any]) -> DATA_FRA
     return df
 
 
-def _reorder_columns_with_upsert_keys_first(
-    df: DATA_FRAME_TYPE, upsert_keys: list[str]
-) -> DATA_FRAME_TYPE:
-    """
-    Reorder a DataFrame's columns so the upsert key columns appear first.
-
-    The upsert keys serve as the row identifiers in the Grid curation UI, so they
-    should always be the leftmost columns of the CSV template. The relative order of
-    the upsert keys is preserved as given, followed by the remaining columns in their
-    original order. Callers are expected to validate that every upsert key is present
-    among the columns before calling this function.
-
-    Args:
-        df: DataFrame whose columns should be reordered.
-        upsert_keys: List of column names to move to the front, in the desired order.
-
-    Returns:
-        DataFrame with the upsert key columns moved to the front.
-    """
-    upsert_key_set = set(upsert_keys)
-    remaining_columns = [
-        column for column in df.columns.tolist() if column not in upsert_key_set
-    ]
-
-    return df[upsert_keys + remaining_columns]
-
-
 def extract_schema_properties_from_web(
     syn: Synapse, schema_uri: str
 ) -> DATA_FRAME_TYPE:
@@ -141,6 +118,7 @@ def create_record_based_metadata_task(
     assignee_principal_id: Optional[Union[str, int]] = None,
     authorization_mode: Optional[Union[AuthorizationMode, str]] = None,
     *,
+    column_order: list[str] | None = None,
     synapse_client: Optional[Synapse] = None,
     project_id: Optional[str] = None,  # Deprecated, will be removed in v5.0.0
     create_grid: bool = True,  # Deprecated, will be removed in v5.0.0
@@ -191,6 +169,36 @@ def create_record_based_metadata_task(
         )
         ```
 
+    Example: Controlling the column order of the RecordSet
+        Pass column_order to place specific columns immediately after the upsert keys.
+        You only need to name the columns you care about; every other schema property
+        is appended afterwards in its existing order. Upsert keys always stay leftmost,
+        so naming one in column_order does not duplicate or move it.
+
+        ```python
+        import synapseclient
+        from synapseclient.extensions.curator import create_record_based_metadata_task
+
+        syn = synapseclient.Synapse()
+        syn.login()
+
+        record_set, curation_task = create_record_based_metadata_task(
+            synapse_client=syn,
+            folder_id="syn87654321",
+            record_set_name="BiospecimenMetadata_RecordSet",
+            record_set_description="RecordSet for biospecimen metadata curation",
+            curation_task_name="BiospecimenMetadataTemplate",
+            upsert_keys=["patientId", "specimenID"],
+            column_order=["diagnosis", "assay"],
+            instructions="Please curate this metadata according to the schema requirements",
+            schema_uri="schema-org-schema.name.schema-v1.0.0",
+            create_grid=False,
+        )
+
+        # Resulting column order:
+        # patientId, specimenID, diagnosis, assay, <remaining schema properties>
+        ```
+
     Arguments:
         folder_id: The Synapse ID of the folder to upload RecordSet to.
         record_set_name: Name for the RecordSet entity that will be created.
@@ -225,6 +233,12 @@ def create_record_based_metadata_task(
             for the current user. Changing this value after the task already exists
             resets the task's active session, so a new grid session must be opened
             before curation can continue.
+        column_order: Optional list of column names placed immediately after the
+            upsert keys, in the order given. Columns that are not named keep their
+            existing relative order and are appended afterwards, so you only need to
+            list the columns that need intentional placement. Naming an upsert key
+            here has no effect, it stays in its leading position. Every name must
+            match a property defined by the schema.
         synapse_client: If not passed in and caching was not disabled by
                 `Synapse.allow_client_caching(False)` this will use the last created
                 instance from the Synapse class constructor.
@@ -239,7 +253,8 @@ def create_record_based_metadata_task(
 
     Raises:
         ValueError: If required parameters are missing, if schema_uri is not provided,
-            or if any upsert_keys are not found among the schema properties.
+            if any upsert_keys are not found among the schema properties, or if
+            column_order is malformed or names a column that is not a schema property.
         SynapseError: If there are issues with Synapse operations.
     """
     # Validate required parameters
@@ -257,6 +272,7 @@ def create_record_based_metadata_task(
         raise ValueError("instructions is required")
     if not schema_uri:
         raise ValueError("schema_uri is required")
+    validate_column_order_list(column_order)
 
     if project_id:
         synapse_client.logger.warning(
@@ -289,9 +305,14 @@ def create_record_based_metadata_task(
             f"{missing_upsert_keys}. Upsert keys identify each row and must correspond "
             "to columns defined in the schema."
         )
-    template_df = _reorder_columns_with_upsert_keys_first(
-        df=template_df, upsert_keys=upsert_keys
-    )
+
+    template_df = template_df[
+        resolve_column_order_list(
+            available_columns=template_df.columns.tolist(),
+            pinned_columns=upsert_keys,
+            requested_columns=column_order,
+        )
+    ]
 
     synapse_client.logger.info(
         f"Extracted schema properties and created template: {template_df.columns.tolist()}"