From 19d043ffba4398a39c6d6b4f242e8e871cfcd79b Mon Sep 17 00:00:00 2001 From: eva-1729 Date: Mon, 11 May 2026 14:46:05 +0100 Subject: [PATCH 1/7] Add GEM Transform --- fia_api/scripts/transforms/factory.py | 3 ++ fia_api/scripts/transforms/gem_transform.py | 58 +++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 fia_api/scripts/transforms/gem_transform.py diff --git a/fia_api/scripts/transforms/factory.py b/fia_api/scripts/transforms/factory.py index 7dadf22b..e123a41e 100644 --- a/fia_api/scripts/transforms/factory.py +++ b/fia_api/scripts/transforms/factory.py @@ -3,6 +3,7 @@ import logging from fia_api.scripts.transforms.enginx_transform import EnginxTransform +from fia_api.scripts.transforms.gem_transform import GEMTransform from fia_api.scripts.transforms.imat_transforms import IMATTransform from fia_api.scripts.transforms.iris_transform import IrisTransform from fia_api.scripts.transforms.mari_transforms import MariTransform @@ -40,6 +41,8 @@ def get_transform_for_instrument(instrument: str) -> Transform: # noqa: PLR0911 return EnginxTransform() case "imat": return IMATTransform() + case "gem": + return GEMTransform() case "test": return TestTransform() case _: diff --git a/fia_api/scripts/transforms/gem_transform.py b/fia_api/scripts/transforms/gem_transform.py new file mode 100644 index 00000000..9baea58e --- /dev/null +++ b/fia_api/scripts/transforms/gem_transform.py @@ -0,0 +1,58 @@ +import logging + +from fia_api.core.models import Job +from fia_api.scripts.pre_script import PreScript + +logger = logging.getLogger(__name__) + +class GEMTransform: + """ + GEMTransform applies modifications to GEM instrument scripts based on reduction input parameters in a Reduction + entity. + """ + + def apply(self, script: PreScript, job: Job) -> None: + logger.info("Beginning GEM transform for job %s...", job.id) + lines = script.value.splitlines() + # MyPY does not believe ColumnElement[JSONB] is indexable, despite JSONB implementing the Indexable mixin + # If you get here in the future, try removing the following line and see if it passes with newer mypy. + + runno = job.inputs["runno"] # type: ignore + if isinstance(runno, list): + if len(runno) > 1: + # Convert list to range string if contiguous, otherwise comma-separated + if all(runno[i] == runno[i - 1] + 1 for i in range(1, len(runno))): + runno_str = f"{runno[0]}-{runno[-1]}" + else: + runno_str = ",".join(map(str, runno)) + else: + runno_str = str(runno[0]) + else: + runno_str = str(runno) + + for index, line in enumerate(lines): + + if line.startswith("mode ="): + lines[index] = f'mode = "{job.inputs["mode"]}"' # type: ignore + continue + if line.startswith("input_mode ="): + lines[index] = f'input_mode = "{job.inputs["input_mode"]}"' # type: ignore + continue + if line.startswith("vanadium_runno ="): + lines[index] = f"vanadium_runno = {vanadium_runno_str}" # type: ignore + continue + if line.startswith("calibration_dir ="): + lines[index] = f"calibration_dir = {job.inputs['calibration_dir']}" # type: ignore + continue + if line.startswith("splined_vanadium_dir ="): + lines[index] = f'splined_vanadium_dir = "{job.inputs["splined_vanadium_dir"]}"' # type: ignore + continue + if line.startswith("config_file ="): + lines[index] = f'config_file = "{job.inputs["config_file"]}"' # type: ignore + continue + if line.startswith("output_dir = "): + lines[index] = f'output_dir = "{job.inputs["output_dir"]}"' # type: ignore + continue + + script.value = "\n".join(lines) + logger.info("Transform complete for job %s", job.id) \ No newline at end of file From 5145ef241cacc51973ba52bed5ab19c7a58cdb77 Mon Sep 17 00:00:00 2001 From: eva-1729 Date: Wed, 10 Jun 2026 09:14:41 +0100 Subject: [PATCH 2/7] Fix small errors plus ruff ignores --- fia_api/scripts/transforms/factory.py | 2 +- fia_api/scripts/transforms/gem_transform.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fia_api/scripts/transforms/factory.py b/fia_api/scripts/transforms/factory.py index e123a41e..05573597 100644 --- a/fia_api/scripts/transforms/factory.py +++ b/fia_api/scripts/transforms/factory.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def get_transform_for_instrument(instrument: str) -> Transform: # noqa: PLR0911 +def get_transform_for_instrument(instrument: str) -> Transform: # noqa: PLR0911, C901 """ Get the appropriate transform for the given instrument and run file :param instrument: str - the instrument diff --git a/fia_api/scripts/transforms/gem_transform.py b/fia_api/scripts/transforms/gem_transform.py index 9baea58e..e4e999fe 100644 --- a/fia_api/scripts/transforms/gem_transform.py +++ b/fia_api/scripts/transforms/gem_transform.py @@ -11,7 +11,7 @@ class GEMTransform: entity. """ - def apply(self, script: PreScript, job: Job) -> None: + def apply(self, script: PreScript, job: Job) -> None: # noqa: C901 logger.info("Beginning GEM transform for job %s...", job.id) lines = script.value.splitlines() # MyPY does not believe ColumnElement[JSONB] is indexable, despite JSONB implementing the Indexable mixin @@ -39,7 +39,10 @@ def apply(self, script: PreScript, job: Job) -> None: lines[index] = f'input_mode = "{job.inputs["input_mode"]}"' # type: ignore continue if line.startswith("vanadium_runno ="): - lines[index] = f"vanadium_runno = {vanadium_runno_str}" # type: ignore + lines[index] = f"vanadium_runno = {runno_str}" # type: ignore + continue + if line.startswith("runno ="): + lines[index] = f"runno = {runno_str}" # type: ignore continue if line.startswith("calibration_dir ="): lines[index] = f"calibration_dir = {job.inputs['calibration_dir']}" # type: ignore From 61faa168b1c5d4887fe36a9dfa3c9103f6141d10 Mon Sep 17 00:00:00 2001 From: eva-1729 Date: Wed, 10 Jun 2026 09:21:17 +0100 Subject: [PATCH 3/7] Add ruff ignore --- fia_api/scripts/transforms/gem_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fia_api/scripts/transforms/gem_transform.py b/fia_api/scripts/transforms/gem_transform.py index e4e999fe..c20c9d29 100644 --- a/fia_api/scripts/transforms/gem_transform.py +++ b/fia_api/scripts/transforms/gem_transform.py @@ -11,7 +11,7 @@ class GEMTransform: entity. """ - def apply(self, script: PreScript, job: Job) -> None: # noqa: C901 + def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 logger.info("Beginning GEM transform for job %s...", job.id) lines = script.value.splitlines() # MyPY does not believe ColumnElement[JSONB] is indexable, despite JSONB implementing the Indexable mixin From 8ceaeaba41e64ad3b598fb1906c57bcae9bb3fb6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 10 Jun 2026 08:22:08 +0000 Subject: [PATCH 4/7] Formatting and linting commit --- fia_api/scripts/transforms/gem_transform.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fia_api/scripts/transforms/gem_transform.py b/fia_api/scripts/transforms/gem_transform.py index c20c9d29..8c58555e 100644 --- a/fia_api/scripts/transforms/gem_transform.py +++ b/fia_api/scripts/transforms/gem_transform.py @@ -5,18 +5,19 @@ logger = logging.getLogger(__name__) + class GEMTransform: """ GEMTransform applies modifications to GEM instrument scripts based on reduction input parameters in a Reduction entity. """ - - def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 + + def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 logger.info("Beginning GEM transform for job %s...", job.id) lines = script.value.splitlines() # MyPY does not believe ColumnElement[JSONB] is indexable, despite JSONB implementing the Indexable mixin # If you get here in the future, try removing the following line and see if it passes with newer mypy. - + runno = job.inputs["runno"] # type: ignore if isinstance(runno, list): if len(runno) > 1: @@ -29,9 +30,8 @@ def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 runno_str = str(runno[0]) else: runno_str = str(runno) - + for index, line in enumerate(lines): - if line.startswith("mode ="): lines[index] = f'mode = "{job.inputs["mode"]}"' # type: ignore continue @@ -58,4 +58,4 @@ def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 continue script.value = "\n".join(lines) - logger.info("Transform complete for job %s", job.id) \ No newline at end of file + logger.info("Transform complete for job %s", job.id) From 56d91cb953f51f65cfc353671c9860e59ea0f9e9 Mon Sep 17 00:00:00 2001 From: eva-1729 Date: Wed, 10 Jun 2026 09:41:04 +0100 Subject: [PATCH 5/7] Add Transform return type and remove 2 type ignores --- fia_api/scripts/transforms/gem_transform.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fia_api/scripts/transforms/gem_transform.py b/fia_api/scripts/transforms/gem_transform.py index 8c58555e..3536a813 100644 --- a/fia_api/scripts/transforms/gem_transform.py +++ b/fia_api/scripts/transforms/gem_transform.py @@ -2,11 +2,12 @@ from fia_api.core.models import Job from fia_api.scripts.pre_script import PreScript +from fia_api.scripts.transforms.transform import Transform logger = logging.getLogger(__name__) -class GEMTransform: +class GEMTransform(Transform): """ GEMTransform applies modifications to GEM instrument scripts based on reduction input parameters in a Reduction entity. @@ -39,10 +40,10 @@ def apply(self, script: PreScript, job: Job) -> None: # noqa: PLR0912,C901 lines[index] = f'input_mode = "{job.inputs["input_mode"]}"' # type: ignore continue if line.startswith("vanadium_runno ="): - lines[index] = f"vanadium_runno = {runno_str}" # type: ignore + lines[index] = f"vanadium_runno = {runno_str}" continue if line.startswith("runno ="): - lines[index] = f"runno = {runno_str}" # type: ignore + lines[index] = f"runno = {runno_str}" continue if line.startswith("calibration_dir ="): lines[index] = f"calibration_dir = {job.inputs['calibration_dir']}" # type: ignore From e382a1a4d413a8759631f7c62501230ede1dc9be Mon Sep 17 00:00:00 2001 From: eva-1729 Date: Fri, 17 Jul 2026 12:02:59 +0100 Subject: [PATCH 6/7] Add tests for gem transform, using Gemini 3.5 Flash --- test/scripts/transforms/test_factory.py | 2 + test/scripts/transforms/test_gem_transform.py | 119 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 test/scripts/transforms/test_gem_transform.py diff --git a/test/scripts/transforms/test_factory.py b/test/scripts/transforms/test_factory.py index 6f1d673e..10524dd9 100644 --- a/test/scripts/transforms/test_factory.py +++ b/test/scripts/transforms/test_factory.py @@ -4,6 +4,7 @@ from fia_api.scripts.transforms.enginx_transform import EnginxTransform from fia_api.scripts.transforms.factory import get_transform_for_instrument +from fia_api.scripts.transforms.gem_transform import GEMTransform from fia_api.scripts.transforms.imat_transforms import IMATTransform from fia_api.scripts.transforms.iris_transform import IrisTransform from fia_api.scripts.transforms.mari_transforms import MariTransform @@ -28,6 +29,7 @@ ("vesuvio", VesuvioTransform), ("enginx", EnginxTransform), ("imat", IMATTransform), + ("gem", GEMTransform), ], ) def test_transform_factory(name, expected_transform): diff --git a/test/scripts/transforms/test_gem_transform.py b/test/scripts/transforms/test_gem_transform.py new file mode 100644 index 00000000..d6da6cb5 --- /dev/null +++ b/test/scripts/transforms/test_gem_transform.py @@ -0,0 +1,119 @@ +"""Test cases for GEMTransform.""" + +from unittest.mock import Mock + +import pytest + +from fia_api.scripts.pre_script import PreScript +from fia_api.scripts.transforms.gem_transform import GEMTransform + +SCRIPT = """ +mode = "default_mode" +input_mode = "default_input_mode" +vanadium_runno = 0 +runno = 0 +calibration_dir = None +splined_vanadium_dir = "default_splined_vanadium_dir" +config_file = "default_config_file" +output_dir = "default_output_dir" +""" + + +@pytest.fixture +def base_job(): + """Fixture for base job inputs.""" + job = Mock() + job.id = "test-job-gem" + job.inputs = { + "mode": "transmission", + "input_mode": "raw", + "calibration_dir": "/path/to/cal", + "splined_vanadium_dir": "/path/to/splined", + "config_file": "/path/to/config", + "output_dir": "/path/to/output", + "runno": 12345, + } + return job + + +@pytest.fixture +def create_expected_script(): + """Fixture returning a helper function to construct the expected script with runno.""" + def _create(runno_str: str) -> str: + return f""" +mode = "transmission" +input_mode = "raw" +vanadium_runno = {runno_str} +runno = {runno_str} +calibration_dir = /path/to/cal +splined_vanadium_dir = "/path/to/splined" +config_file = "/path/to/config" +output_dir = "/path/to/output\"""" + return _create + + +def test_gem_transform_single_run(base_job, create_expected_script): + """Test GEMTransform with a single run number.""" + script = PreScript(value=SCRIPT) + GEMTransform().apply(script, base_job) + + assert script.value == create_expected_script("12345") + + +def test_gem_transform_contiguous_runs(base_job, create_expected_script): + """Test GEMTransform with contiguous runs.""" + base_job.inputs["runno"] = [12345, 12346, 12347] + script = PreScript(value=SCRIPT) + GEMTransform().apply(script, base_job) + + assert script.value == create_expected_script("12345-12347") + + +def test_gem_transform_non_contiguous_runs(base_job, create_expected_script): + """Test GEMTransform with non-contiguous runs.""" + base_job.inputs["runno"] = [12345, 12347, 12349] + script = PreScript(value=SCRIPT) + GEMTransform().apply(script, base_job) + + assert script.value == create_expected_script("12345,12347,12349") + + +def test_gem_transform_list_length_one(base_job, create_expected_script): + """Test GEMTransform with list containing a single run.""" + base_job.inputs["runno"] = [12345] + script = PreScript(value=SCRIPT) + GEMTransform().apply(script, base_job) + + assert script.value == create_expected_script("12345") + + +def test_gem_transform_apply(base_job): + """Test GEMTransform only modifies expected lines and leaves others unchanged.""" + transform = GEMTransform() + script = PreScript(value=SCRIPT) + original_lines = script.value.splitlines() + + transform.apply(script, base_job) + + updated_lines = script.value.splitlines() + assert len(original_lines) == len(updated_lines) + + for index, line in enumerate(updated_lines): + if line.startswith("mode ="): + assert line == 'mode = "transmission"' + elif line.startswith("input_mode ="): + assert line == 'input_mode = "raw"' + elif line.startswith("vanadium_runno ="): + assert line == "vanadium_runno = 12345" + elif line.startswith("runno ="): + assert line == "runno = 12345" + elif line.startswith("calibration_dir ="): + assert line == "calibration_dir = /path/to/cal" + elif line.startswith("splined_vanadium_dir ="): + assert line == 'splined_vanadium_dir = "/path/to/splined"' + elif line.startswith("config_file ="): + assert line == 'config_file = "/path/to/config"' + elif line.startswith("output_dir = "): + assert line == 'output_dir = "/path/to/output"' + else: + assert line == original_lines[index] From cd4266f1ead1115ef85554b5a197e5ad70dd7546 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 17 Jul 2026 11:03:44 +0000 Subject: [PATCH 7/7] Formatting and linting commit --- test/scripts/transforms/test_gem_transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/scripts/transforms/test_gem_transform.py b/test/scripts/transforms/test_gem_transform.py index d6da6cb5..f1205f55 100644 --- a/test/scripts/transforms/test_gem_transform.py +++ b/test/scripts/transforms/test_gem_transform.py @@ -39,6 +39,7 @@ def base_job(): @pytest.fixture def create_expected_script(): """Fixture returning a helper function to construct the expected script with runno.""" + def _create(runno_str: str) -> str: return f""" mode = "transmission" @@ -49,6 +50,7 @@ def _create(runno_str: str) -> str: splined_vanadium_dir = "/path/to/splined" config_file = "/path/to/config" output_dir = "/path/to/output\"""" + return _create