Skip to content

fix: data process and processing schema creation#40

Draft
arjunsridhar12345 wants to merge 2 commits into
devfrom
37-fix-data-process-creation
Draft

fix: data process and processing schema creation#40
arjunsridhar12345 wants to merge 2 commits into
devfrom
37-fix-data-process-creation

Conversation

@arjunsridhar12345

Copy link
Copy Markdown
Collaborator

Attempts to fix #37. Adds the pipeline name to the data process object so it can be merged later in the metadata manager. Adding the pipeline name to the data process requires the same name to be present in the list of pipelines as well. Thus, that has been added here as well.

Since the pipeline is now semantically versioned, this version and the version in the nextflow config should be the same. Sample output is shown below trying to test this fix:

{
   "object_type": "Processing",
   "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/processing.py",
   "schema_version": "2.3.0",
   "data_processes": [
      {
         "object_type": "Data process",
         "process_type": "Other",
         "name": "Other",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "",
            "name": null,
            "version": "",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T01:15:30Z",
         "output_path": null,
         "output_parameters": null,
         "notes": "Data was copied",
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Compression",
         "name": "Compression",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "ghcr.io/allenneuraldynamics/aind-behavior-video-transformation",
            "name": null,
            "version": "0.2.1",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior-videos",
               "output_directory": "/allen/aind/stage/svc_aind_airflow/prod/860898_2026-07-09_19-33-53_aa0568f31b/behavior-videos"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T11:36:16Z",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Pipeline",
         "name": "Dynamic foraging NWB packaging",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
            "name": null,
            "version": "0.1.0",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": null,
            "core_dependency": null
         },
         "experimenters": [
            "Alex Piet",
            "Micah Woodard",
            "Bruno Cruz",
            "Arjun Sridhar"
         ],
         "pipeline_name": "dynamic-foraging-processing-pipeline",
         "start_date_time": "2026-07-14T10:28:04.143454-07:00",
         "end_date_time": "2026-07-14T10:28:04.143454-07:00",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      }
   ],
   "pipelines": [
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
         "name": "dynamic-foraging-processing-pipeline",
         "version": "0.1.0",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": null,
         "language_version": null,
         "input_data": null,
         "parameters": null,
         "core_dependency": null
      },
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/aind-airflow-dags",
         "name": "transform_and_upload_v2",
         "version": "0.39.1",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": "Python",
         "language_version": "3.9.18",
         "input_data": null,
         "parameters": {
            "job_type": "vr_foraging_v2"
         },
         "core_dependency": null
      }
   ],
   "notes": "Processes were reordered by start_date_time",
   "dependency_graph": {
      "Dynamic foraging NWB packaging": [],
      "Other": [],
      "Compression": []
   }
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant