diff --git a/datasets/hls2/dataset.yaml b/datasets/hls2/dataset.yaml index 10a24fd2..4e870314 100644 --- a/datasets/hls2/dataset.yaml +++ b/datasets/hls2/dataset.yaml @@ -20,7 +20,10 @@ collections: splits: - depth: 1 options: - ends_with: .jpg + # Key on _stac.json, not .jpg: LP DAAC stopped publishing the browse + # image for HLSS30 around 2026-06-17, which made every later granule + # invisible to enumeration. _stac.json is present on S30 and L30. + ends_with: _stac.json min_depth: 6 max_depth: 6 chunk_length: 20000 @@ -36,7 +39,8 @@ collections: splits: - depth: 1 options: - ends_with: .jpg + # Kept in sync with hls2-s30 above; see the note there. + ends_with: _stac.json min_depth: 6 max_depth: 6 chunk_length: 20000 diff --git a/datasets/hls2/hls2.py b/datasets/hls2/hls2.py index ce6e517f..5449b81b 100644 --- a/datasets/hls2/hls2.py +++ b/datasets/hls2/hls2.py @@ -18,8 +18,13 @@ logger.setLevel(logging.INFO) -# regex for the jpg blob path -hls2_regex = re.compile(r"([SL]30)/(\d{2})/([A-Z])/([A-Z]{2})/(\d{4})/(\d{2})/(\d{2})/HLS.[SL]30.T(\d{2})([A-Z]{3}).(\d{7})T(\d{6}).v2.0/.*\.jpg") +# regex for the STAC metadata blob path +hls2_regex = re.compile( + r"([SL]30)/(\d{2})/([A-Z])/([A-Z]{2})/(\d{4})/(\d{2})/(\d{2})/" + r"HLS.[SL]30.T(\d{2})([A-Z]{3}).(\d{7})T(\d{6}).v2.0/.*_stac\.json" +) + +STAC_JSON_SUFFIX = "_stac.json" # Band assets (does not include thumbnail) S30_assets = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12', 'Fmask', 'SZA', 'SAA', 'VZA', 'VAA'] @@ -30,20 +35,25 @@ class HLS2Collection(Collection): def create_item( cls, asset_uri: str, storage_factory: StorageFactory, upload: bool = True, ) -> Union[List[pystac.Item], WaitTaskResult]: - # The asset_uri is the full blob uri path to the .jpg file - storage, thumbnail_path = storage_factory.get_storage_for_file(asset_uri) + # The asset_uri is the full blob uri path to the _stac.json file. + # It used to be the .jpg browse image, but LP DAAC stopped publishing + # that for HLSS30 around 2026-06-17. + storage, stac_json_path = storage_factory.get_storage_for_file(asset_uri) - if not storage.file_exists(thumbnail_path): - raise Exception(f"{thumbnail_path} does not exist in {storage}") + if not storage.file_exists(stac_json_path): + raise Exception(f"{stac_json_path} does not exist in {storage}") - m = re.match(hls2_regex, thumbnail_path) + m = re.match(hls2_regex, stac_json_path) if not m: - raise Exception(f"{thumbnail_path} did not match regex") + raise Exception(f"{stac_json_path} did not match regex") + + # ".../HLS.S30.T06WXC.2026213T212519.v2.0" - every sibling asset is + # this prefix plus a suffix (".B01.tif", ".jpg", ...). + base_path = stac_json_path[: -len(STAC_JSON_SUFFIX)] logger.debug("Reading existing STAC Item JSON from NASA") - + # Download STAC Item JSON provided by NASA - stac_json_path = thumbnail_path.replace('.jpg', '_stac.json') with TemporaryDirectory() as tmp_dir: nc_name = os.path.basename(stac_json_path) tmp_nc_path = os.path.join(tmp_dir, nc_name) @@ -54,33 +64,38 @@ def create_item( logger.error(f"Failed to read in STAC Item from {tmp_nc_path}: {e}") return [] - # Verify all assets exist in blob (we already know the thumbnail exists) + # Verify all band assets exist in blob if 'sentinel' in item.properties["platform"]: - for S30_asset in S30_assets: - if not storage.file_exists(thumbnail_path.replace('.jpg', f'.{S30_asset}.tif')): - logger.error(f"{S30_asset} does not exist in {storage}") - return [] + expected_assets = S30_assets elif 'landsat' in item.properties["platform"]: - for L30_asset in L30_assets: - if not storage.file_exists(thumbnail_path.replace('.jpg', f'.{L30_asset}.tif')): - logger.error(f"{L30_asset} does not exist in {storage}") - return [] + expected_assets = L30_assets else: logger.error(f"Unknown platform {item.properties['platform']}") return [] + for expected_asset in expected_assets: + if not storage.file_exists(f"{base_path}.{expected_asset}.tif"): + logger.error(f"{expected_asset} does not exist in {storage}") + return [] + # Clear out links TODO: DO WE NEED ANY LINKS ADDED HERE? item.links = [] + # The browse image is optional - HLSS30 no longer ships one. + has_thumbnail = storage.file_exists(f"{base_path}.jpg") + # Update the hrefs and remove the stac JSON from the asset list - thumbnail_uri_https = f"{storage.account_url}/{storage.container_name}/{thumbnail_path}" - for asset in item.assets: + base_uri_https = f"{storage.account_url}/{storage.container_name}/{base_path}" + for asset in list(item.assets): if '.tif' in item.assets[asset].href: - item.assets[asset].href = thumbnail_uri_https.replace('.jpg', f'.{asset}.tif') + item.assets[asset].href = f"{base_uri_https}.{asset}.tif" # TODO - make sure this exists in blob elif asset == 'thumbnail': - item.assets[asset].href = thumbnail_uri_https # we've already checked this exists - elif '_stac.json' in item.assets[asset].href: + if has_thumbnail: + item.assets[asset].href = f"{base_uri_https}.jpg" + else: + item.assets.pop(asset) + elif STAC_JSON_SUFFIX in item.assets[asset].href: item.assets.pop(asset) # Some of the json we already copied had its extensions cleared, so add them back in diff --git a/datasets/hls2/test_hls2.py b/datasets/hls2/test_hls2.py index eff43961..af1b94e3 100644 --- a/datasets/hls2/test_hls2.py +++ b/datasets/hls2/test_hls2.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize( "asset_uri", [ - f"blob://{test_storage_account}/{test_container}/L30/05/U/LB/2025/02/26/HLS.L30.T05ULB.2025057T212139.v2.0/HLS.L30.T05ULB.2025057T212139.v2.0.jpg" + f"blob://{test_storage_account}/{test_container}/L30/05/U/LB/2025/02/26/HLS.L30.T05ULB.2025057T212139.v2.0/HLS.L30.T05ULB.2025057T212139.v2.0_stac.json" ], ) def test_hls2_landsat(asset_uri: str) -> None: @@ -27,7 +27,7 @@ def test_hls2_landsat(asset_uri: str) -> None: @pytest.mark.parametrize( "asset_uri", [ - f"blob://{test_storage_account}/{test_container}/S30/05/M/KR/2025/01/05/HLS.S30.T05MKR.2025005T204621.v2.0/HLS.S30.T05MKR.2025005T204621.v2.0.jpg" + f"blob://{test_storage_account}/{test_container}/S30/05/M/KR/2025/01/05/HLS.S30.T05MKR.2025005T204621.v2.0/HLS.S30.T05MKR.2025005T204621.v2.0_stac.json" ], ) def test_hls2_sentinel(asset_uri: str) -> None: @@ -41,4 +41,22 @@ def test_hls2_sentinel(asset_uri: str) -> None: assert item.properties["platform"] == "sentinel-2a" assert len(item.assets) == 19 # 18 bands and 1 thumbnail assert "thumbnail" in item.assets - item.validate() \ No newline at end of file + item.validate() + + +@pytest.mark.parametrize( + "asset_uri", + [ + f"blob://{test_storage_account}/{test_container}/S30/06/W/XC/2026/08/01/HLS.S30.T06WXC.2026213T212519.v2.0/HLS.S30.T06WXC.2026213T212519.v2.0_stac.json" # noqa: E501 + ], +) +def test_hls2_sentinel_without_browse_image(asset_uri: str) -> None: + """LP DAAC stopped publishing the .jpg browse image for HLSS30 around 2026-06-17.""" + result = hls2.HLS2Collection.create_item(asset_uri, StorageFactory(), upload=False) + assert result + item, = result + assert isinstance(item, Item) + assert item.properties["platform"].startswith("sentinel-2") + assert len(item.assets) == 18 # 18 bands, no thumbnail + assert "thumbnail" not in item.assets + item.validate()