Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions workflow/envs/environment.yaml
Comment thread
irr-github marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ dependencies:
# Dependencies of the workflow itself
- xlrd
- libgdal-hdf5
- libgdal-netcdf
- rioxarray
- openpyxl
- pycountry
Expand Down
116 changes: 30 additions & 86 deletions workflow/notebooks/compare.ipynb

Large diffs are not rendered by default.

34 changes: 6 additions & 28 deletions workflow/notebooks/land_availability.ipynb

Large diffs are not rendered by default.

427 changes: 43 additions & 384 deletions workflow/notebooks/validate.ipynb

Large diffs are not rendered by default.

88 changes: 61 additions & 27 deletions workflow/scripts/calculate_gebco_slope.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,44 @@
Requires: GDAL (gdalwarp, gdaldem)
"""

import subprocess
import os
import sys
import logging
import os
import shutil
import subprocess
from pathlib import Path

from _helpers import mock_snakemake

# Set up logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)


def check_gdal_availability():
"""Check if GDAL tools are available in the system.

Raises:
RuntimeError: If required GDAL tools are not found.
"""
required_tools = ["gdalwarp", "gdaldem"]
missing_tools = []

for tool in required_tools:
if not shutil.which(tool):
missing_tools.append(tool)

if missing_tools:
error_msg = (
f"Required GDAL tools not found: {', '.join(missing_tools)}\n"
f"Please install GDAL. If using conda: conda install -c conda-forge gdal\n"
f"Or ensure GDAL is installed and available in your PATH."
)
logger.error(error_msg)
raise RuntimeError(error_msg)

logger.info(f"✓ GDAL tools available: {', '.join(required_tools)}")


def setup_proj_environment():
"""Set up PROJ_LIB environment variable for GDAL to find projection database."""
if "PROJ_LIB" not in os.environ:
Expand Down Expand Up @@ -60,13 +87,27 @@ def run_command(cmd: str, description: str):

if result.returncode != 0:
logger.error(f"Command failed with return code {result.returncode}")
logger.error(f"STDERR: {result.stderr}")
raise RuntimeError(f"{description} failed")
logger.error(f"Command: {cmd}")

# Log stdout if present (some tools output errors to stdout)
if result.stdout:
logger.error(f"STDOUT:\n{result.stdout}")

# Log stderr with better formatting
if result.stderr:
logger.error(f"STDERR:\n{result.stderr}")
else:
logger.error("No error output captured")

raise RuntimeError(
f"{description} failed: {result.stderr.strip() if result.stderr else 'Unknown error'}"
)

if result.stdout:
logger.debug(f"STDOUT: {result.stdout}")
if result.stderr:
logger.warning(f"STDERR: {result.stderr}")
# GDAL often outputs progress info to stderr, so only warn if there's content
logger.debug(f"STDERR: {result.stderr}")

logger.info(f"✓ {description} completed successfully")
return result
Expand Down Expand Up @@ -98,6 +139,9 @@ def calculate_slope(input_gebco, output_slope, threads=4, log_file=None):
# Create output directory if needed
output_slope.parent.mkdir(parents=True, exist_ok=True)

# Check GDAL availability
check_gdal_availability()

# Set up PROJ environment for GDAL
setup_proj_environment()

Expand Down Expand Up @@ -125,11 +169,7 @@ def calculate_slope(input_gebco, output_slope, threads=4, log_file=None):
run_command(cmd1, "Step 1: Reproject to Mollweide (ESRI:54009)")
logger.info("Calculating slope...")
# Step 2: Calculate slope in percent
cmd2 = (
f"gdaldem slope -p "
f"-of netCDF -co FORMAT=NC4 "
f"{mollweide_file} {slope_mollweide_file}"
)
cmd2 = f"gdaldem slope -p -of netCDF -co FORMAT=NC4 {mollweide_file} {slope_mollweide_file}"
run_command(cmd2, "Step 2: Calculate slope (percent)")

# Step 3: Reproject back to EPSG:4326 with compression
Expand Down Expand Up @@ -162,19 +202,13 @@ def calculate_slope(input_gebco, output_slope, threads=4, log_file=None):

if __name__ == "__main__":
# When called from snakemake
if "snakemake" in globals():
calculate_slope(
input_gebco=snakemake.input.gebco,
output_slope=snakemake.output.slope,
threads=snakemake.threads,
log_file=snakemake.log[0] if snakemake.log else None,
)
# When called from command line
elif len(sys.argv) >= 3:
input_file = sys.argv[1]
output_file = sys.argv[2]
threads = int(sys.argv[3]) if len(sys.argv) > 3 else 4
calculate_slope(input_file, output_file, threads)
else:
print("Usage: python calculate_gebco_slope.py <input_gebco> <output_slope> [threads]")
sys.exit(1)
if "snakemake" not in globals():
snakemake = mock_snakemake("calculate_gebco_slope")

calculate_slope(
input_gebco=snakemake.input.gebco,
output_slope=snakemake.output.slope,
threads=snakemake.threads,
log_file=snakemake.log[0] if snakemake.log else None,
)
logger.info("GEBCO slope calculation script finished.")
Loading