diff --git a/alert_system/etl/base/extraction.py b/alert_system/etl/base/extraction.py index 13efd26a6..c0c910211 100644 --- a/alert_system/etl/base/extraction.py +++ b/alert_system/etl/base/extraction.py @@ -229,11 +229,10 @@ def _extract_event_items(self, extraction_run_id: str, is_past_event: bool = Fal logger.warning(f"Failed to fetch events: {e}") raise - first_event_item = next(event_items, None) - if not first_event_item: - msg = f"No event items found for extraction_run_id={extraction_run_id}" - logger.warning(msg) - raise ValueError(msg) + event_items = list(event_items) + if not event_items: + logger.warning(f"No event items found for extraction_run_id={extraction_run_id}") + return for event_item in event_items: self.process_event_item(event_item=event_item, extraction_run_id=extraction_run_id, is_past_event=is_past_event) diff --git a/alert_system/helpers.py b/alert_system/helpers.py index 20c646dc9..751062ec1 100644 --- a/alert_system/helpers.py +++ b/alert_system/helpers.py @@ -1,7 +1,32 @@ # Helper functions to build search params. from typing import Dict, Generator, Optional +from urllib.parse import urlsplit, urlunsplit import httpx +from django.conf import settings + + +def _remap_stac_url(url: str) -> str: + """ + Rewrite a STAC URL to point at the internal cluster service. + """ + external_base = getattr(settings, "EOAPI_STAC_EXTERNAL_URL", None) + internal_base = getattr(settings, "EOAPI_STAC_INTERNAL_URL", None) + if not external_base or not internal_base: + return url + + ext = urlsplit(external_base.rstrip("/")) + internal = urlsplit(internal_base.rstrip("/")) + parsed = urlsplit(url) + + if parsed.netloc not in (ext.netloc, internal.netloc): + return url + + path = parsed.path + if ext.path and path.startswith(ext.path): + path = path[len(ext.path) :] + + return urlunsplit((internal.scheme, internal.netloc, internal.path + path, parsed.query, parsed.fragment)) def build_search_params( @@ -57,7 +82,7 @@ def build_stac_search( def fetch_stac_data(url: str, payload: dict | None = None, timeout: int | None = 60): - response = httpx.get(url=url, params=payload, timeout=timeout) + response = httpx.get(url=_remap_stac_url(url), params=payload, timeout=timeout) response.raise_for_status() return response.json() diff --git a/main/settings.py b/main/settings.py index a500bef8c..8fdf2af06 100644 --- a/main/settings.py +++ b/main/settings.py @@ -151,6 +151,9 @@ # PowerBI POWERBI_WORKSPACE_ID=(str, None), POWERBI_DATASET_IDS=(str, None), + # Alert system - remap external STAC related-item URLs to internal cluster URLs + EOAPI_STAC_EXTERNAL_URL=(str, None), # e.g. https://montandon-eoapi.ifrc.org/stac + EOAPI_STAC_INTERNAL_URL=(str, None), # e.g. http://montandon-eoapi-stac.montandon-eoapi.svc.cluster.local:8080 ) @@ -174,6 +177,9 @@ def parse_domain(*env_keys: str) -> str: return domain.strip("/") +EOAPI_STAC_EXTERNAL_URL = env("EOAPI_STAC_EXTERNAL_URL") +EOAPI_STAC_INTERNAL_URL = env("EOAPI_STAC_INTERNAL_URL") + GO_API_URL = parse_domain("API_FQDN") GO_WEB_URL = parse_domain("FRONTEND_URL") # NOTE: Used in local development to point to the frontend service from within go-api container