Serve low-res volumes from a writable root; fix make_lowres for real dataset layout

flask-server/api/api_blueprint.py

@@ -45,6 +45,14 @@
 api_blueprint = Blueprint("api", __name__)
 last_session_check = datetime.now()
 
+# Low-res volumes (generated by scripts/make_lowres.py) live on a WRITABLE disk,
+# NOT the read-only dataset mount. The tree mirrors the dataset:
+#   <LOWRES_ROOT>/image_only/<case>/ct_lowres.nii.gz
+#   <LOWRES_ROOT>/mask_only/<case>/combined_labels_lowres.nii.gz
+# Overridable via PANTS_LOWRES_PATH. If a file is absent the API serves full res,
+# so this stays fully additive (no-op until the batch has run).
+LOWRES_ROOT = os.environ.get("PANTS_LOWRES_PATH", "/home/visitor/pants_lowres")
+
 import hmac
 import threading
 
@@ -344,9 +352,11 @@ def get_main_nifti(clabel_id):
     main_nifti_path = f"{case_dir}/{Constants.MAIN_NIFTI_FILENAME}"
 
     # ?res=low → serve the precomputed low-res copy when present (much smaller/faster
-    # for big full-body scans). Falls back to full res if it hasn't been generated.
+    # for big full-body scans). It lives under LOWRES_ROOT (a writable disk), NOT the
+    # read-only dataset mount. Falls back to full res if it hasn't been generated.
     if (request.args.get('res') or '').strip().lower() == 'low':
-        low_path = f"{case_dir}/{Constants.MAIN_NIFTI_FILENAME.replace('.nii.gz', '_lowres.nii.gz')}"
+        low_name = Constants.MAIN_NIFTI_FILENAME.replace('.nii.gz', '_lowres.nii.gz')
+        low_path = f"{LOWRES_ROOT}/image_only/{get_panTS_id(secure_filename(clabel_id))}/{low_name}"
         if os.path.exists(low_path):
             main_nifti_path = low_path
 
@@ -746,9 +756,11 @@ async def get_segmentations(combined_labels_id):
     nifti_path = f"{Constants.PANTS_PATH}/mask_only/{get_panTS_id(secure_filename(combined_labels_id))}/{Constants.COMBINED_LABELS_NIFTI_FILENAME}"
     labels = list(Constants.PREDEFINED_LABELS.values())
     # ?res=low → serve the precomputed low-res mask (paired with the low-res CT so the
-    # overlay stays aligned). Falls back to full res below if it hasn't been generated.
+    # overlay stays aligned). It lives under LOWRES_ROOT (a writable disk), NOT the
+    # read-only mask_only mount. Falls back to full res below if it hasn't been generated.
     if (request.args.get('res') or '').strip().lower() == 'low':
-        low_path = nifti_path.replace('.nii.gz', '_lowres.nii.gz')
+        low_name = Constants.COMBINED_LABELS_NIFTI_FILENAME.replace('.nii.gz', '_lowres.nii.gz')
+        low_path = f"{LOWRES_ROOT}/mask_only/{get_panTS_id(secure_filename(combined_labels_id))}/{low_name}"
         if os.path.exists(low_path):
             response = make_response(send_file(low_path, mimetype='application/gzip'))
             response.headers["Cross-Origin-Resource-Policy"] = "cross-origin"

flask-server/scripts/make_lowres.py

@@ -5,22 +5,27 @@
 only on demand ("HD"). Meant to run on the JHU server, which has the data + the
 hardware — it's CPU/disk only, no new infrastructure.
 
-Per case it writes, next to the originals:
-    ct.nii.gz              -> ct_lowres.nii.gz               (linear, order=1)
-    combined_labels.nii.gz -> combined_labels_lowres.nii.gz  (nearest, order=0)
+The dataset mount is READ-ONLY, so the low-res copies are written under a separate
+writable --out-root (default /home/visitor/pants_lowres, or $PANTS_LOWRES_PATH),
+mirroring the dataset layout so the API can find them:
+    <out_root>/image_only/<case>/ct_lowres.nii.gz               (linear, order=1)
+    <out_root>/mask_only/<case>/combined_labels_lowres.nii.gz    (nearest, order=0)
 
 CT and segmentation are downsampled by the SAME --factor so they stay geometrically
 aligned (the viewer overlays them) — at low res the relationship is identical to full
 res, so the overlay can't break. Both files are written together per case (or neither),
 so the API never ends up serving a low-res CT against a full-res mask.
 
 Fully additive + reversible: if the *_lowres.nii.gz files are absent, the API serves
-the originals exactly as before. Delete the low-res files to revert.
-
-Usage (on the server):
-    cd flask-server && venv/bin/python scripts/make_lowres.py --factor 2
-    # quick trial run:
-    venv/bin/python scripts/make_lowres.py --factor 2 --limit 5
+the originals exactly as before. Delete the --out-root tree to revert.
+
+Usage (on the server — use the conda python that runs the backend):
+    PYBIN=/home/visitor/.conda/envs/PanTS_backend/bin/python3.11
+    cd flask-server
+    # quick trial run first:
+    $PYBIN scripts/make_lowres.py --factor 2 --limit 5
+    # then the full batch (long-running — run under nohup/tmux):
+    nohup $PYBIN scripts/make_lowres.py --factor 2 > /tmp/make_lowres.log 2>&1 &
 Idempotent: skips cases that already have low-res files unless --overwrite.
 """
 import argparse
@@ -54,20 +59,25 @@ def _downsample(img, factor, order):
 
 
 def _seg_path_for(ct_path):
-    """Map .../ImageTr/<id>/ct.nii.gz -> .../LabelTr/<id>/combined_labels.nii.gz."""
+    """Map .../image_only/<id>/ct.nii.gz -> .../mask_only/<id>/combined_labels.nii.gz."""
     p = ct_path.replace(f"{os.sep}image_only{os.sep}", f"{os.sep}mask_only{os.sep}")
     return p.replace(CT_NAME, SEG_NAME)
 
 
-def _process_case(ct_path, factor, overwrite):
+def _process_case(ct_path, out_root, factor, overwrite):
     seg_path = _seg_path_for(ct_path)
     if not os.path.exists(seg_path):
         return "no_seg"  # skip entirely so we never pair low CT with full mask
 
-    ct_low = os.path.join(os.path.dirname(ct_path), CT_LOW)
-    seg_low = os.path.join(os.path.dirname(seg_path), SEG_LOW)
+    # Write under out_root, mirroring image_only/<case>/ + mask_only/<case>/ so the
+    # API (LOWRES_ROOT) can find them. The dataset mount itself is read-only.
+    case_id = os.path.basename(os.path.dirname(ct_path))
+    ct_low = os.path.join(out_root, "image_only", case_id, CT_LOW)
+    seg_low = os.path.join(out_root, "mask_only", case_id, SEG_LOW)
     if os.path.exists(ct_low) and os.path.exists(seg_low) and not overwrite:
         return "skip"
+    os.makedirs(os.path.dirname(ct_low), exist_ok=True)
+    os.makedirs(os.path.dirname(seg_low), exist_ok=True)
 
     # CT: linear interpolation, keep int16 Hounsfield units.
     ct_img = nib.load(ct_path)
@@ -87,8 +97,11 @@ def _process_case(ct_path, factor, overwrite):
 
 
 def main():
+    default_out = os.environ.get("PANTS_LOWRES_PATH", "/home/visitor/pants_lowres")
     ap = argparse.ArgumentParser(description="Generate low-res CT + seg copies.")
     ap.add_argument("--factor", type=float, default=2.0, help="downsample factor per axis (>=1)")
+    ap.add_argument("--out-root", default=default_out,
+                    help=f"writable dir for low-res output (default: {default_out})")
     ap.add_argument("--overwrite", action="store_true", help="regenerate even if low-res exists")
     ap.add_argument("--limit", type=int, default=0, help="process at most N cases (0 = all)")
     args = ap.parse_args()
@@ -98,19 +111,20 @@ def main():
     if args.factor < 1:
         sys.exit("--factor must be >= 1")
 
-    root = os.path.join(Constants.PANTS_PATH, "data")
-    ct_paths = []
-    for sub in ("ImageTr", "ImageTe"):
-        ct_paths += sorted(glob.glob(os.path.join(root, sub, "*", CT_NAME)))
+    # CT volumes live under <PANTS_PATH>/image_only/<case>/ct.nii.gz (masks are in the
+    # parallel mask_only/ tree — see _seg_path_for).
+    root = os.path.join(Constants.PANTS_PATH, "image_only")
+    ct_paths = sorted(glob.glob(os.path.join(root, "*", CT_NAME)))
     if args.limit:
         ct_paths = ct_paths[: args.limit]
 
     print(f"Found {len(ct_paths)} CT volumes under {root} (factor={args.factor})")
+    print(f"Writing low-res copies under {args.out_root}")
     counts = {"ok": 0, "skip": 0, "no_seg": 0, "err": 0}
     t0 = time.time()
     for i, ct in enumerate(ct_paths, 1):
         try:
-            result = _process_case(ct, args.factor, args.overwrite)
+            result = _process_case(ct, args.out_root, args.factor, args.overwrite)
         except Exception as e:  # never let one bad file abort the batch
             result = "err"
             print(f"  [err] {ct}: {e}")