diff --git a/deep_code/cli/generate_config.py b/deep_code/cli/generate_config.py index ce1c286..ba63d0d 100644 --- a/deep_code/cli/generate_config.py +++ b/deep_code/cli/generate_config.py @@ -18,5 +18,5 @@ help="Output directory for templates", ) def generate_config(output_dir): - TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow_config.yaml") - TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset_config.yaml") + TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow.yaml") + TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset.yaml") diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index ac306f5..dfc8295 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -15,7 +15,6 @@ @click.group() def main(): """Deep Code CLI.""" - pass main.add_command(publish) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 6ccc8eb..4048b09 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -24,7 +24,7 @@ def setUp(self, mock_github_publisher, mock_fsspec_open): # Mock dataset and workflow config files self.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.workflow_config = { "properties": {"title": "Test Workflow"}, @@ -94,7 +94,7 @@ def test_read_config_files(self): # Mock dataset and workflow config files dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } workflow_config = { "properties": {"title": "Test Workflow"}, @@ -138,7 +138,7 @@ def test_publish_mode_routing(self, mock_wf, mock_ds, mock_s3): self.publisher.dataset_config = { "stac_catalog_s3_root": "s3://bucket/stac/", "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.publisher.gh_publisher.publish_files.return_value = "PR_URL" @@ -198,7 +198,6 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds, mock_s3): assert "dataset: col" in kwargs["pr_title"] assert "workflow/experiment: wf" in kwargs["pr_title"] - # ------------------------------------------------------------------ # S3 credential resolution # ------------------------------------------------------------------ @@ -262,9 +261,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"}, "s3://bucket/col/item.json": {"type": "Feature", "id": "item"}, } - self.publisher._write_stac_catalog_to_s3( - file_dict, {"key": "k", "secret": "s"} - ) + self.publisher._write_stac_catalog_to_s3(file_dict, {"key": "k", "secret": "s"}) self.assertEqual(mock_fsspec_open.call_count, 2) mock_fsspec_open.assert_any_call( @@ -283,9 +280,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): def test_publish_writes_zarr_stac_to_s3_when_configured( self, mock_publish_ds, mock_fsspec_open ): - self.publisher.dataset_config["stac_catalog_s3_root"] = ( - "s3://test-bucket/stac/" - ) + self.publisher.dataset_config["stac_catalog_s3_root"] = "s3://test-bucket/stac/" mock_ctx = MagicMock() mock_ctx.__enter__ = MagicMock(return_value=MagicMock()) @@ -295,7 +290,9 @@ def test_publish_writes_zarr_stac_to_s3_when_configured( mock_generator = MagicMock() mock_generator.build_zarr_stac_catalog_file_dict.return_value = { "s3://test-bucket/stac/catalog.json": {"type": "Catalog"}, - "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"}, + "s3://test-bucket/stac/test-collection/items/test-collection.json": { + "type": "Feature" + }, } # Simulate what publish_dataset() normally does: store the generator self.publisher._last_generator = mock_generator @@ -330,8 +327,10 @@ def test_publish_dataset_creates_project_collection_when_missing( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -340,9 +339,14 @@ def test_publish_dataset_creates_project_collection_when_missing( # Project collection is missing; all other file_exists calls return True self.publisher.gh_publisher.github_automation.file_exists.return_value = False - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - file_dict = self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + file_dict = self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_called_once() self.assertIn("projects/test-project/collection.json", file_dict) @@ -365,8 +369,10 @@ def test_publish_dataset_updates_project_collection_when_exists( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -375,9 +381,14 @@ def test_publish_dataset_updates_project_collection_when_exists( # Project collection already exists self.publisher.gh_publisher.github_automation.file_exists.return_value = True - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_not_called() @@ -388,7 +399,9 @@ def test_publish_dataset_updates_project_collection_when_exists( def test_publish_dataset_raises_when_stac_root_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", } with pytest.raises(ValueError, match="stac_catalog_s3_root"): @@ -400,14 +413,24 @@ def test_publish_dataset_raises_when_no_dataset_config(self): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_ids_missing(self): - self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="Dataset ID or Collection ID missing"): + self.publisher.dataset_config = { + "collection_id": "", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "license_type": "CC-BY-4.0", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", + } + with pytest.raises(ValueError, match="collection_id missing"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } with pytest.raises(ValueError, match="license_type is required"): self.publisher.publish_dataset(write_to_file=False) @@ -431,14 +454,18 @@ def test_update_and_add_to_file_dict(self): file_dict = {} self.publisher.gh_publisher.github_automation.local_clone_dir = "/tmp" update_method = MagicMock(return_value={"key": "value"}) - self.publisher._update_and_add_to_file_dict(file_dict, "some/catalog.json", update_method) + self.publisher._update_and_add_to_file_dict( + file_dict, "some/catalog.json", update_method + ) update_method.assert_called_once() assert any("some/catalog.json" in str(k) for k in file_dict) def test_update_variable_catalogs_creates_new_when_missing(self): mock_gen = MagicMock() mock_gen.variables_metadata = {"var1": {"variable_id": "var1"}} - mock_gen.build_variable_catalog.return_value.to_dict.return_value = {"id": "var1"} + mock_gen.build_variable_catalog.return_value.to_dict.return_value = { + "id": "var1" + } self.publisher.gh_publisher.github_automation.file_exists.return_value = False file_dict = {} @@ -486,7 +513,7 @@ def _setup_workflow_mocks(self): @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF): - mock_rg, mock_props, mock_wf_record, _ = self._setup_workflow_mocks() + mock_rg, _mock_props, mock_wf_record, _ = self._setup_workflow_mocks() MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record @@ -507,8 +534,12 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF @patch("deep_code.tools.publish.WorkflowAsOgcRecord") @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") - def test_generate_workflow_records_mode_all(self, MockRG, MockLinks, MockWF, MockExp): - mock_rg, mock_props, mock_wf_record, mock_exp_record = self._setup_workflow_mocks() + def test_generate_workflow_records_mode_all( + self, MockRG, MockLinks, MockWF, MockExp + ): + mock_rg, _mock_props, mock_wf_record, mock_exp_record = ( + self._setup_workflow_mocks() + ) MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record MockExp.return_value = mock_exp_record @@ -583,8 +614,8 @@ class TestParseGithubNotebookUrl: ], ) def test_valid_urls(self, url, repo_url, repo_name, branch, file_path): - got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url( - url + got_repo_url, got_repo_name, got_branch, got_file_path = ( + LinksBuilder._parse_github_notebook_url(url) ) assert got_repo_url == repo_url assert got_repo_name == repo_name diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index e981df6..9b0a2d7 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -8,7 +8,7 @@ import tempfile import unittest from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np from pystac import Catalog, Collection, Item @@ -24,12 +24,15 @@ VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator, Theme +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + Theme, +) class TestOSCProductSTACGenerator(unittest.TestCase): - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_data_store): + def setUp(self): """Set up a mock dataset and generator.""" self.mock_dataset = Dataset( coords={ @@ -65,18 +68,27 @@ def setUp(self, mock_data_store): ), }, ) - mock_store = MagicMock() - mock_store.open_data.return_value = self.mock_dataset - mock_data_store.return_value = self.mock_dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.mock_dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.generator = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", - access_link="s3://mock-bucket/mock-dataset", + access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", + osc_project="deep-earth-system-data-lab", osc_status="ongoing", osc_region="Global", osc_themes=["climate", "environment"], @@ -84,18 +96,19 @@ def setUp(self, mock_data_store): def test_open_dataset(self): """Test if the dataset is opened correctly.""" - self.assertIsInstance(self.generator.dataset, Dataset) + dataset = self.mock_dataset + self.assertIsInstance(dataset, Dataset) for coord in ("lon", "lat", "time"): - self.assertIn(coord, self.generator.dataset.coords) + self.assertIn(coord, dataset.coords) def test_get_spatial_extent(self): """Test spatial extent extraction.""" - extent = self.generator._get_spatial_extent() + extent = self.generator._get_spatial_extent(self.mock_dataset) self.assertEqual(extent.bboxes[0], [-180.0, -90.0, 180.0, 90.0]) def test_get_temporal_extent(self): """Test temporal extent extraction.""" - extent = self.generator._get_temporal_extent() + extent = self.generator._get_temporal_extent(self.mock_dataset) # TemporalExtent.intervals is a list of [start, end] interval = extent.intervals[0] self.assertEqual(interval[0], datetime(2023, 1, 1, 0, 0)) @@ -103,12 +116,12 @@ def test_get_temporal_extent(self): def test_get_variables(self): """Test variable ID extraction.""" - vars_ = self.generator.get_variable_ids() + vars_ = self.generator.get_variable_ids(self.mock_dataset) self.assertCountEqual(vars_, ["var1", "var2"]) def test_get_general_metadata(self): """Test general metadata extraction.""" - meta = self.generator._get_general_metadata() + meta = self.generator._get_general_metadata(self.mock_dataset) self.assertEqual(meta.get("description"), "Mock dataset for testing.") def test_extract_metadata_for_variable(self): @@ -121,7 +134,7 @@ def test_extract_metadata_for_variable(self): def test_get_variables_metadata(self): """Test metadata dict for all variables.""" - meta_dict = self.generator.get_variables_metadata() + meta_dict = self.generator.get_variables_metadata(self.mock_dataset) self.assertIn("var1", meta_dict) self.assertIn("var2", meta_dict) self.assertIsInstance(meta_dict["var1"], dict) @@ -134,11 +147,10 @@ def test_build_theme(self): ids = [tc.id for tc in theme_obj.concepts] self.assertListEqual(ids, ["a", "b"]) - @patch.object(OscDatasetStacGenerator, "_add_gcmd_link_to_var_catalog") - @patch.object(OscDatasetStacGenerator, "add_themes_as_related_links_var_catalog") - def test_build_variable_catalog(self, mock_add_themes, mock_add_gcmd): + def test_build_variable_catalog(self): """Test building of variable-level STAC catalog.""" - var_meta = self.generator.variables_metadata["var1"] + variables_metadata = self.generator.get_variables_metadata(self.mock_dataset) + var_meta = variables_metadata["var1"] catalog = self.generator.build_variable_catalog(var_meta) self.assertIsInstance(catalog, Catalog) self.assertEqual(catalog.id, "var1") @@ -162,12 +174,10 @@ def test_update_product_base_catalog(self): } ], } - import tempfile import json as _json + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -202,13 +212,11 @@ def test_update_variable_base_catalog(self): } ], } - import tempfile import json as _json import os + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -219,9 +227,7 @@ def test_update_variable_base_catalog(self): os.unlink(tmp_path) self.assertIsInstance(result, dict) - child_hrefs = [ - lnk["href"] for lnk in result["links"] if lnk["rel"] == "child" - ] + child_hrefs = [lnk["href"] for lnk in result["links"] if lnk["rel"] == "child"] self.assertEqual(len(child_hrefs), len(vars_)) # self link must remain in place self.assertEqual(result["links"][0]["rel"], "self") @@ -239,8 +245,13 @@ def test_osc_project_custom(self, mock_open_ds): """A custom osc_project is stored on the generator.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -287,8 +298,13 @@ def test_build_project_collection_custom_project(self, mock_open_ds): """build_project_collection reflects a custom osc_project.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -389,13 +405,11 @@ def test_update_deepesdl_collection(self): } ], } - import tempfile import json as _json import os + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -419,7 +433,9 @@ def test_update_deepesdl_collection(self): def test_build_zarr_stac_item_structure(self): """Item has correct geometry, bbox, datetime range, assets, and links.""" s3_root = "s3://test-bucket/stac/my-collection/" - item = self.generator.build_zarr_stac_item(s3_root) + item = self.generator.build_zarr_stac_item( + self.generator.items_config[0], s3_root + ) self.assertIsInstance(item, Item) self.assertEqual(item.id, "mock-collection-id") @@ -443,13 +459,14 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset") + self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( - meta_asset.href, "s3://mock-bucket/mock-dataset/.zmetadata" + meta_asset.href, + "s3://mock-bucket/mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) @@ -479,8 +496,12 @@ def test_build_zarr_stac_item_structure(self): def test_build_zarr_stac_item_trailing_slash_normalised(self): """Trailing slash on s3_root should not produce double slashes.""" - item_with = self.generator.build_zarr_stac_item("s3://bucket/stac/") - item_without = self.generator.build_zarr_stac_item("s3://bucket/stac") + item_with = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac/" + ) + item_without = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac" + ) self.assertEqual(item_with.self_href, item_without.self_href) def test_build_zarr_stac_catalog_file_dict_keys(self): @@ -490,7 +511,8 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -506,7 +528,8 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -514,6 +537,40 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertIn("zarr-data", item_dict["assets"]) self.assertIn("zarr-consolidated-metadata", item_dict["assets"]) + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): + """Only the first item configuration is emitted in the STAC file dict.""" + mock_open_ds.return_value = self.mock_dataset + gen = OscDatasetStacGenerator( + collection_id="multi-collection", + items_config=[ + ItemConfig(dataset_id="first.zarr", item_id="first-item"), + ItemConfig(dataset_id="second.zarr", item_id="second-item"), + ], + workflow_id="dummy", + workflow_title="test", + license_type="proprietary", + osc_project="deep-earth-system-data-lab", + ) + + file_dict = gen.build_zarr_stac_catalog_file_dict( + "s3://test-bucket/stac/multi-collection/" + ) + + self.assertIn( + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/first-item.json", + file_dict, + ) + self.assertNotIn( + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/second-item.json", + file_dict, + ) + catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] + item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] + self.assertEqual(len(item_links), 1) + def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added when stac_catalog_s3_root is provided. @@ -526,21 +583,31 @@ def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): collection = self.generator.build_dataset_stac_collection( mode="dataset", stac_catalog_s3_root=s3_root ) - https_catalog = "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + https_catalog = ( + "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + ) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" + https_catalog.replace("https://", "") ) via_link = next( - (lnk for lnk in collection.links if lnk.rel == "via" and "stac-browser" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "via" and "stac-browser" in str(lnk.target) + ), None, ) self.assertIsNotNone(via_link, "Expected a 'via' STAC browser link") self.assertEqual(via_link.target, stac_browser_href) child_link = next( - (lnk for lnk in collection.links if lnk.rel == "child" and "catalog.json" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "child" and "catalog.json" in str(lnk.target) + ), None, ) self.assertIsNotNone(child_link, "Expected a 'child' HTTPS catalog link") @@ -618,20 +685,27 @@ def test_edge_cases(self): class TestOscDatasetStacGeneratorExtra(unittest.TestCase): """Additional tests to cover branches not exercised by TestOSCProductSTACGenerator.""" - def _make_generator(self, mock_ds, collection_id="my-collection", **kwargs): - with patch("deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds): + @staticmethod + def _make_generator(mock_ds, collection_id="my-collection", **kwargs): + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds + ): + kwargs.setdefault("osc_project", "deep-earth-system-data-lab") return OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id=collection_id, + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id=collection_id), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", **kwargs, ) - def _make_dataset(self, coord_type="lon_lat"): - import numpy as np + @staticmethod + def _make_dataset(coord_type="lon_lat"): from datetime import datetime + if coord_type == "lon_lat": coords = { "lon": ("lon", np.linspace(-10, 10, 3)), @@ -653,6 +727,7 @@ def _make_dataset(self, coord_type="lon_lat"): else: coords = {} from xarray import Dataset + return Dataset(coords=coords) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -660,11 +735,14 @@ def test_collection_id_with_space_raises(self, mock_open_ds): mock_open_ds.return_value = self._make_dataset() with self.assertRaisesRegex(ValueError, "must not contain spaces"): OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="bad id", + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id="bad-id"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -672,7 +750,7 @@ def test_spatial_extent_longitude_latitude(self, mock_open_ds): ds = self._make_dataset("longitude_latitude") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], -10.0) self.assertAlmostEqual(extent.bboxes[0][1], -5.0) @@ -681,7 +759,7 @@ def test_spatial_extent_x_y(self, mock_open_ds): ds = self._make_dataset("x_y") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], 0.0) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -690,7 +768,7 @@ def test_spatial_extent_unknown_coords_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "recognized spatial coordinates"): - gen._get_spatial_extent() + gen._get_spatial_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_temporal_extent_no_time_raises(self, mock_open_ds): @@ -698,7 +776,7 @@ def test_temporal_extent_no_time_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "time"): - gen._get_temporal_extent() + gen._get_temporal_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_normalize_name_none_returns_none(self, mock_open_ds): @@ -710,9 +788,14 @@ def test_normalize_name_none_returns_none(self, mock_open_ds): def test_build_collection_with_cf_params(self, mock_open_ds): ds = self._make_dataset() mock_open_ds.return_value = ds - gen = self._make_generator(ds, cf_params=[{"name": "temperature", "units": "K"}]) + gen = self._make_generator( + ds, cf_params=[{"name": "temperature", "units": "K"}] + ) collection = gen.build_dataset_stac_collection(mode="dataset") - self.assertEqual(collection.extra_fields.get("cf:parameter"), [{"name": "temperature", "units": "K"}]) + self.assertEqual( + collection.extra_fields.get("cf:parameter"), + [{"name": "temperature", "units": "K"}], + ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_collection_with_visualisation_link(self, mock_open_ds): @@ -731,11 +814,12 @@ def test_build_collection_mode_all_adds_experiment_link(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) collection = gen.build_dataset_stac_collection(mode="all") - exp_links = [lnk for lnk in collection.links if "experiments" in str(lnk.target)] + exp_links = [ + lnk for lnk in collection.links if "experiments" in str(lnk.target) + ] self.assertEqual(len(exp_links), 1) - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def test_s3_to_https(self, mock_open_ds): + def test_s3_to_https(self): self.assertEqual( OscDatasetStacGenerator._s3_to_https("s3://my-bucket/path/to/file.json"), "https://my-bucket.s3.amazonaws.com/path/to/file.json", @@ -762,7 +846,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): json.dump(base, f) tmp_path = f.name try: - result = gen.update_existing_variable_catalog(tmp_path, "var1") + result = gen.update_existing_variable_catalog(tmp_path) finally: os.unlink(tmp_path) @@ -774,8 +858,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): class TestPRRCollection(unittest.TestCase): """Tests for the PRR-style Collection -> Item -> Assets generation.""" - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_open_ds): + def setUp(self): self.dataset = Dataset( coords={ "lon": ("lon", np.linspace(-20, 20, 4)), @@ -804,14 +887,22 @@ def setUp(self, mock_open_ds): }, attrs={"description": "PRR test cube"}, ) - mock_open_ds.return_value = self.dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + osc_project="deep-earth-system-data-lab", + access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -821,8 +912,8 @@ def setUp(self, mock_open_ds): # ---- helpers ---- - def test_get_epsg_from_spatial_ref(self): - self.assertEqual(self.gen._get_epsg(), 3035) + def test_get_crs_from_spatial_ref(self): + self.assertEqual(self.gen._get_crs(self.dataset).to_epsg(), 3035) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_get_epsg_default_4326(self, mock_open_ds): @@ -836,16 +927,19 @@ def test_get_epsg_default_4326(self, mock_open_ds): ) mock_open_ds.return_value = ds gen = OscDatasetStacGenerator( - dataset_id="t.zarr", collection_id="c", + items_config=[ + ItemConfig(dataset_id="t.zarr", item_id="c"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) - self.assertEqual(gen._get_epsg(), 4326) + self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) def test_get_cube_dimensions(self): - dims = self.gen._get_cube_dimensions() + dims = self.gen._get_cube_dimensions(self.dataset) self.assertEqual(set(dims), {"lon", "lat", "time"}) self.assertEqual(dims["lon"]["type"], "spatial") self.assertEqual(dims["lon"]["axis"], "x") @@ -857,7 +951,7 @@ def test_get_cube_dimensions(self): self.assertEqual(len(dims["time"]["extent"]), 2) def test_get_cube_variables(self): - variables = self.gen._get_cube_variables() + variables = self.gen._get_cube_variables(self.dataset) # CRS variable must be excluded. self.assertEqual(set(variables), {"sst", "chl"}) self.assertEqual(variables["sst"]["type"], "data") @@ -871,7 +965,7 @@ def test_get_cube_variables(self): # ---- item ---- def test_build_prr_stac_item(self): - item = self.gen.build_prr_stac_item() + item = self.gen.build_prr_stac_item(self.gen.items_config[0]) self.assertIsInstance(item, Item) self.assertEqual(item.id, "prr-collection") self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) @@ -932,16 +1026,19 @@ def test_build_prr_collection_conformant_fields(self): return_value=self.dataset, ): gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket/", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], osc_missions=["sentinel-3"], + osc_project="deep-earth-system-data-lab", osc_project_description="A detailed project description.", osc_project_website="https://project.example.org", osc_contract_number="4000114410/15/NL/BW", @@ -975,7 +1072,7 @@ def test_build_prr_collection_fallbacks(self): self.assertEqual(ef["osc:initiative"], "earthcode") # Fallbacks: website -> documentation_link, description -> dataset description. self.assertEqual(ef["osc:project_website"], "https://example.org/doc") - self.assertEqual(ef["osc:project_description"], "PRR test cube") + self.assertEqual(ef["osc:project_description"], "No description provided.") # No thumbnail / contract number configured -> absent. self.assertNotIn("thumbnail", coll.assets) self.assertNotIn("osc:contract-number", ef) @@ -1001,12 +1098,15 @@ def test_build_prr_collection_cf_params_override(self): def test_build_prr_collection_no_themes(self, mock_open_ds): mock_open_ds.return_value = self.dataset gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + osc_project="deep-earth-system-data-lab", + access_link_root="s3://bucket", ) coll = gen.build_prr_collection() self.assertNotIn("themes", coll.extra_fields) @@ -1018,8 +1118,10 @@ def test_save_prr_collection_writes_tree(self): out = self.gen.save_prr_collection(tmp) self.assertEqual(out, tmp) - collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + collection_path = os.path.join(tmp, "prr-collection", "collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) self.assertTrue(os.path.isfile(collection_path)) self.assertTrue(os.path.isfile(item_path)) @@ -1033,9 +1135,7 @@ def test_save_prr_collection_writes_tree(self): self.assertEqual(item_dict["type"], "Feature") # Structural links are relative; the Item link points at the child. - item_link = next( - lnk for lnk in coll_dict["links"] if lnk["rel"] == "item" - ) + item_link = next(lnk for lnk in coll_dict["links"] if lnk["rel"] == "item") self.assertFalse(item_link["href"].startswith("s3://")) self.assertTrue(item_link["href"].endswith(".json")) @@ -1047,7 +1147,9 @@ def test_save_prr_collection_writes_tree(self): def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: self.gen.save_prr_collection(tmp) - coll = Collection.from_file(os.path.join(tmp, "collection.json")) + coll = Collection.from_file( + os.path.join(tmp, "prr-collection", "collection.json") + ) items = list(coll.get_items()) self.assertEqual(len(items), 1) self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/tests/utils/test_github_automation.py b/deep_code/tests/utils/test_github_automation.py index 671adb3..25ca33b 100644 --- a/deep_code/tests/utils/test_github_automation.py +++ b/deep_code/tests/utils/test_github_automation.py @@ -47,6 +47,7 @@ def test_clone_sync_repository_new(self, mock_run): """ No .git directory → we clone and then ensure upstream remote gets added. """ + # Simulate: "git remote -v" returns nothing so we add 'upstream' def run_side_effect(args, cwd, check, capture_output=False, text=True): if args[:3] == ["git", "remote", "-v"]: @@ -55,8 +56,9 @@ def run_side_effect(args, cwd, check, capture_output=False, text=True): mock_run.side_effect = run_side_effect - with patch.object(Path, "mkdir") as _mk, patch( - "pathlib.Path.exists", side_effect=lambda p=None: False + with ( + patch.object(Path, "mkdir") as _mk, + patch("pathlib.Path.exists", side_effect=lambda p=None: False), ): self.gha.clone_sync_repository() @@ -239,9 +241,10 @@ def test_create_branch(self, mock_run): @patch("subprocess.run") def test_add_file(self, mock_run): mock_run.return_value = make_cp() - with patch.object(Path, "mkdir") as _mk, patch.object( - Path, "write_text" - ) as _wt: + with ( + patch.object(Path, "mkdir") as _mk, + patch.object(Path, "write_text") as _wt, + ): # Ensure .git exists with patch( "pathlib.Path.exists", diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d910db0..5558501 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -4,14 +4,13 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -from typing import Optional import yaml class TemplateGenerator: @staticmethod - def generate_workflow_template(output_path: Optional[str] = None) -> str: + def generate_workflow_template(output_path: str | None = None) -> str: """Generate a complete template with all possible keys and placeholder values""" workflow_template = { @@ -48,32 +47,48 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: with open(output_path, "w") as f: f.write("# Workflow Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") - f.write(yaml.dump(workflow_template, sort_keys=False, width=1000, - default_flow_style=False)) + f.write( + yaml.dump( + workflow_template, + sort_keys=False, + width=1000, + default_flow_style=False, + ) + ) @staticmethod - def generate_dataset_template(output_path: Optional[str] = None) -> str: + def generate_dataset_template(output_path: str | None = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { - "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", "collection_id": "[REQUIRED: unique identifier, no spaces — use hyphens (e.g. My-Dataset-2024)]", "license_type": "[REQUIRED: SPDX license identifier, e.g. CC-BY-4.0, MIT, proprietary]", "stac_catalog_s3_root": "[REQUIRED: S3 root for the STAC Catalog + Item, e.g. s3://my-bucket/stac/my-collection/]", + "items_config": [ + { + "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", + "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", + } + ], + "osc_project": "[REQUIRED: OSC project ID (e.g. deep-earth-system-data-lab)]", + "osc_project_url": "[REQUIRED: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", - "osc_themes": ["[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]"], + "osc_themes": [ + "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" + ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", - "access_link": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public/{dataset_id}]", - "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], + "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]", + "cf_parameter": [ + {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} + ], } # Fields used only by `deep-code generate-prr-collection` to build a @@ -85,7 +100,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", - "osc_missions": ["[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]"], + "osc_missions": [ + "[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]" + ], "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", @@ -95,7 +112,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" - "# {stac_catalog_s3_root}/{collection_id}/item.json (STAC Item for the whole Zarr)\n" + "# {stac_catalog_s3_root}/{collection_id}/items/{item_id}.json (STAC Item for each Zarr)\n" + "# items_config can contain multiple dataset/item pairs, but publish\n" + "# currently only consumes one item configuration.\n" "# S3 write credentials are resolved in order:\n" "# 1. STAC_S3_KEY / STAC_S3_SECRET env vars (STAC-specific, any bucket)\n" "# 2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars\n" @@ -107,11 +126,23 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write("# Dataset Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") f.write("# --- REQUIRED fields ---\n") - f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + required, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write("\n# --- OPTIONAL fields ---\n") - f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + optional, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write( "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" ) - f.write(yaml.dump(prr, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + prr, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write(stac_catalog_comment) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 9bfddac..6a65035 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -16,7 +16,7 @@ import fsspec import yaml -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator logger = logging.getLogger(__name__) @@ -38,29 +38,42 @@ def generate_prr_collection( with fsspec.open(dataset_config_path, "r") as file: config = yaml.safe_load(file) or {} - dataset_id = config.get("dataset_id") collection_id = config.get("collection_id") + osc_project = config.get("osc_project") + osc_project_url = config.get("osc_project_url") license_type = config.get("license_type") - - if not dataset_id or not collection_id: - raise ValueError( - "Both 'dataset_id' and 'collection_id' are required in the dataset config." - ) + items_config_raw = config.get("items_config") + if not collection_id: + raise ValueError("collection_id is required in the dataset config.") + if not osc_project: + raise ValueError("osc_project is required in the dataset config.") + if not osc_project_url: + raise ValueError("osc_project_url is required in the dataset config.") if not license_type: raise ValueError( "license_type is required in the dataset config. " "Provide an SPDX identifier (e.g. 'CC-BY-4.0', 'MIT', 'proprietary')." ) + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] logger.info(f"Generating PRR STAC collection for '{collection_id}'.") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, collection_id=collection_id, + items_config=items_config, workflow_id=config.get("workflow_id") or "", workflow_title=config.get("workflow_title") or "", license_type=license_type, documentation_link=config.get("documentation_link"), - access_link=config.get("access_link"), + collection_title=config.get("collection_title"), + access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", osc_themes=config.get("osc_themes"), @@ -68,7 +81,8 @@ def generate_prr_collection( cf_params=config.get("cf_parameter"), visualisation_link=config.get("visualisation_link"), description=config.get("description"), - **({"osc_project": config["osc_project"]} if config.get("osc_project") else {}), + osc_project=config.get("osc_project"), + osc_project_title=config.get("osc_project_title"), osc_project_url=config.get("osc_project_url"), # PRR-specific project metadata. osc_initiative=config.get("osc_initiative") or "earthcode", @@ -81,6 +95,6 @@ def generate_prr_collection( sci_citation=config.get("sci_citation"), ) - out_dir = output_dir or config.get("prr_output_dir") or f"prr/{collection_id}" + out_dir = output_dir or config.get("prr_output_dir") or "prr" generator.save_prr_collection(out_dir) return out_dir diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 3b62cbd..399f102 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,11 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + open_dataset, +) from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -160,6 +164,8 @@ def __init__( # Values that may be set from configs self.collection_id: str | None = None + self.osc_project: str | None = None + self.osc_project_url: str | None = None self.workflow_title: str | None = None self.workflow_id: str | None = None @@ -245,9 +251,24 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): / var_file_path ) file_dict[var_file_path] = generator.update_existing_variable_catalog( - full_path, var_id + full_path ) + @staticmethod + def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: + """Build item configs from the dataset config.""" + items_config_raw = dataset_config.get("items_config") + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + return items_config + def publish_dataset( self, write_to_file: bool = False, @@ -260,23 +281,31 @@ def publish_dataset( raise ValueError( "No dataset config loaded. Provide dataset_config_path to publish dataset." ) - dataset_id = self.dataset_config.get("dataset_id") + items_config = self._build_items_config(self.dataset_config) + if len(items_config) != 1: + raise ValueError( + "publish currently supports exactly one item configuration." + ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - access_link = self.dataset_config.get("access_link") + access_link_root = self.dataset_config.get("access_link_root") dataset_status = self.dataset_config.get("dataset_status") or "ongoing" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") cf_params = self.dataset_config.get("cf_parameter") license_type = self.dataset_config.get("license_type") visualisation_link = self.dataset_config.get("visualisation_link") - osc_project = self.dataset_config.get("osc_project") + self.osc_project = self.dataset_config.get("osc_project") osc_project_title = self.dataset_config.get("osc_project_title") - osc_project_url = self.dataset_config.get("osc_project_url") + self.osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") - if not dataset_id or not self.collection_id: - raise ValueError("Dataset ID or Collection ID missing in the config.") + if not self.collection_id: + raise ValueError("collection_id missing in the config.") + if not self.osc_project: + raise ValueError("osc_project missing in the config.") + if not self.osc_project_url: + raise ValueError("osc_project missing in the config.") if not license_type: raise ValueError( @@ -295,27 +324,28 @@ def publish_dataset( logger.info("Generating STAC collection...") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=self.collection_id, workflow_id=self.workflow_id, workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - access_link=access_link, + access_link_root=access_link_root, osc_status=dataset_status, osc_region=osc_region, osc_themes=osc_themes, cf_params=cf_params, visualisation_link=visualisation_link, - **({"osc_project": osc_project} if osc_project else {}), + osc_project=self.osc_project, osc_project_title=osc_project_title, - osc_project_url=osc_project_url, + osc_project_url=self.osc_project_url, description=description, ) # Store so publish() can reuse it for zarr STAC catalog generation self._last_generator = generator - variable_ids = generator.get_variable_ids() + dataset = open_dataset(generator.items_config[0].dataset_id) + variable_ids = generator.get_variable_ids(dataset) ds_collection = generator.build_dataset_stac_collection( mode=mode, stac_catalog_s3_root=stac_catalog_s3_root ) @@ -352,7 +382,9 @@ def publish_dataset( file_dict[project_collection_path] = generator.build_project_collection() # Add child link in the projects base catalog self._update_and_add_to_file_dict( - file_dict, "projects/catalog.json", generator.update_project_base_catalog + file_dict, + "projects/catalog.json", + generator.update_project_base_catalog, ) else: self._update_and_add_to_file_dict( diff --git a/deep_code/tools/test.py b/deep_code/tools/test.py index 5bdf092..b42e21f 100644 --- a/deep_code/tools/test.py +++ b/deep_code/tools/test.py @@ -1,2 +1,2 @@ -""" Execute the application package of a published experiment on a subset of input data +"""Execute the application package of a published experiment on a subset of input data to verify the reproducibility is achieved""" diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 6deaa3c..373029b 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -5,9 +5,14 @@ import json import logging +from dataclasses import dataclass from datetime import datetime, timezone +from typing import Any +import numpy as np import pandas as pd +import pyproj +import xarray as xr from pystac import ( Asset, Catalog, @@ -35,13 +40,21 @@ from deep_code.utils.osc_extension import OscExtension +@dataclass +class ItemConfig: + dataset_id: str + item_id: str + + class OscDatasetStacGenerator: """Generates OSC STAC Collections for a product from Zarr datasets. Args: - dataset_id: ID of the Zarr dataset. collection_id: Unique identifier for the STAC collection. - access_link: Public access link to the dataset. + items_config: List of item configuration entries. Each item maps one + dataset_id to one item_id + collection_title: Title present in the collection and in the STAC browser + access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -53,20 +66,21 @@ class OscDatasetStacGenerator: def __init__( self, - dataset_id: str, collection_id: str, + items_config: list[ItemConfig], workflow_id: str, workflow_title: str, license_type: str, - access_link: str | None = None, + osc_project: str, + collection_title: str | None = None, + access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", osc_region: str = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, - cf_params: list[dict[str]] | None = None, - osc_project: str = "deep-earth-system-data-lab", - osc_project_title: str = "DeepESDL", + cf_params: list[dict[str, Any]] | None = None, + osc_project_title: str | None = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, @@ -82,21 +96,25 @@ def __init__( if " " in collection_id: raise ValueError( f"collection_id must not contain spaces: {collection_id!r}. " - "Use hyphens as word separators (e.g. 'My-Dataset-2024')." + "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - self.dataset_id = dataset_id self.collection_id = collection_id + self.items_config = items_config self.workflow_id = workflow_id self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project - self.osc_project_title = osc_project_title + self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url - self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}" + self.access_link_root = access_link_root or "s3://deep-esdl-public/" + self.collection_title = collection_title or collection_id self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region - self.osc_themes = [t.lower() for t in (osc_themes or [])] + if osc_themes is None: + osc_themes = [] + assert isinstance(osc_themes, list) + self.osc_themes = [t.lower() for t in osc_themes] self.osc_missions = osc_missions or [] self.cf_params = cf_params or {} self.visualisation_link = visualisation_link @@ -111,55 +129,46 @@ def __init__( self.sci_doi = sci_doi self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) - self.variables_metadata = self.get_variables_metadata() - - def _get_spatial_extent(self) -> SpatialExtent: - """Extract spatial extent from the dataset.""" - if {"lon", "lat"}.issubset(self.dataset.coords): - # For regular gridding - lon_min, lon_max = ( - float(self.dataset.lon.min()), - float(self.dataset.lon.max()), - ) - lat_min, lat_max = ( - float(self.dataset.lat.min()), - float(self.dataset.lat.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"longitude", "latitude"}.issubset(self.dataset.coords): - # For regular gridding with 'longitude' and 'latitude' - lon_min, lon_max = ( - float(self.dataset.longitude.min()), - float(self.dataset.longitude.max()), - ) - lat_min, lat_max = ( - float(self.dataset.latitude.min()), - float(self.dataset.latitude.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"x", "y"}.issubset(self.dataset.coords): - # For irregular gridding - x_min, x_max = (float(self.dataset.x.min()), float(self.dataset.x.max())) - y_min, y_max = (float(self.dataset.y.min()), float(self.dataset.y.max())) - return SpatialExtent([[x_min, y_min, x_max, y_max]]) + + def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: + """Extract the spatial extent and return it in EPSG:4326.""" + + if {"lon", "lat"}.issubset(dataset.coords): + x_name, y_name = "lon", "lat" + elif {"longitude", "latitude"}.issubset(dataset.coords): + x_name, y_name = "longitude", "latitude" + elif {"x", "y"}.issubset(dataset.coords): + x_name, y_name = "x", "y" else: raise ValueError( "Dataset does not have recognized spatial coordinates " - "('lon', 'lat' or 'x', 'y')." + "('lon', 'lat'), ('longitude', 'latitude'), or ('x', 'y')." + ) + + x_min = float(dataset[x_name].min()) + x_max = float(dataset[x_name].max()) + y_min = float(dataset[y_name].min()) + y_max = float(dataset[y_name].max()) + + crs = self._get_crs(dataset) + + if crs.to_epsg() != 4326: + transformer = pyproj.Transformer.from_crs(crs, 4326, always_xy=True) + x_min, y_min, x_max, y_max = transformer.transform_bounds( + x_min, y_min, x_max, y_max ) - def _get_temporal_extent(self) -> TemporalExtent: + return SpatialExtent([[x_min, y_min, x_max, y_max]]) + + @staticmethod + def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" - if "time" in self.dataset.coords: + dataset = dataset + if "time" in dataset.coords: try: # Convert the time bounds to datetime objects - time_min = pd.to_datetime( - self.dataset.time.min().values - ).to_pydatetime() - time_max = pd.to_datetime( - self.dataset.time.max().values - ).to_pydatetime() + time_min = pd.to_datetime(dataset.time.min().values).to_pydatetime() + time_max = pd.to_datetime(dataset.time.max().values).to_pydatetime() return TemporalExtent([[time_min, time_max]]) except Exception as e: raise ValueError(f"Failed to parse temporal extent: {e}") @@ -172,11 +181,47 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - def _get_general_metadata(self) -> dict: + def _build_access_link(self, item_config: ItemConfig) -> str: + """Return the asset href for an item, supporting prefix and full URLs.""" + root = self.access_link_root + if root.endswith("/"): + root = root.rstrip("/") + return f"{root}/{item_config.dataset_id}" + + @staticmethod + def _union_spatial_extent(items: list[Item]) -> SpatialExtent: + """Merge multiple dataset spatial extents into a single bounding box.""" + bboxes = [item.bbox for item in items] + return SpatialExtent( + [ + [ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ] + ] + ) + + @staticmethod + def _union_temporal_extent(items: list[Item]) -> TemporalExtent: + """Merge multiple dataset temporal extents into a single interval.""" + intervals = np.array( + [ + [ + datetime.fromisoformat(item.properties["start_datetime"]), + datetime.fromisoformat(item.properties["end_datetime"]), + ] + for item in items + ] + ) + return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) + + def _get_general_metadata(self, dataset: xr.Dataset) -> dict: return { "description": ( self.description - or self.dataset.attrs.get("description") + or dataset.attrs.get("description") or "No description available." ) } @@ -194,19 +239,19 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: """Get variable IDs for all variables in the dataset.""" - variable_ids = list(self.variables_metadata.keys()) + variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. return [ var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata(self) -> dict[str, dict]: + def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" variables_metadata = {} - for var_name, variable in self.dataset.data_vars.items(): + for variable in dataset.data_vars.values(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata @@ -445,7 +490,9 @@ def build_project_collection(self) -> dict: CONTACTS_SCHEMA_URI, ], "title": self.format_string(self.osc_project_title or self.osc_project), - "description": self.format_string(self.osc_project_title or self.osc_project), + "description": self.format_string( + self.osc_project_title or self.osc_project + ), "keywords": [], "license": "various", "extent": { @@ -502,7 +549,7 @@ def update_deepesdl_collection(self, deepesdl_collection_full_path) -> dict: ) return data - def update_existing_variable_catalog(self, var_file_path, var_id) -> dict: + def update_existing_variable_catalog(self, var_file_path) -> dict: """Append child and theme links to an existing variable catalog.""" with open(var_file_path, encoding="utf-8") as f: data = json.load(f) @@ -536,7 +583,7 @@ def _s3_to_https(s3_url: str) -> str: Example: s3://my-bucket/path/to/file → https://my-bucket.s3.amazonaws.com/path/to/file """ - without_scheme = s3_url[len("s3://"):] + without_scheme = s3_url[len("s3://") :] bucket, _, key = without_scheme.partition("/") return f"https://{bucket}.s3.amazonaws.com/{key}" @@ -549,40 +596,50 @@ def format_string(s: str) -> str: @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) - def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: + def build_zarr_stac_item( + self, + item_config: ItemConfig, + stac_catalog_s3_root: str, + ) -> Item: """Build a single STAC Item representing the entire Zarr store. One item covers the full spatiotemporal extent of the dataset. Assets point to the Zarr store and its consolidated metadata. Args: + item_config: object containing `dataset_id` and `item_id` stac_catalog_s3_root: S3 root URL where the STAC catalog will be hosted (e.g. ``s3://my-bucket/stac/``). Used to build self/root/parent hrefs. Returns: A :class:`pystac.Item` ready to be serialised to S3. """ - self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + self.logger.info( + f"Building STAC Item {item_config.item_id} " + f"for collection '{self.collection_id}'." + ) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -602,11 +659,12 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, properties={ + "title": self.format_string(item_config.item_id), "start_datetime": start_dt.isoformat() if start_dt else None, "end_datetime": end_dt.isoformat() if end_dt else None, "description": general_metadata.get("description", ""), @@ -616,26 +674,39 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item.collection_id = self.collection_id item.set_self_href(item_href) - item.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - item.add_link(Link(rel="parent", target=catalog_href, media_type="application/json")) - item.add_link(Link( - rel="collection", - target=osc_collection_href, - media_type="application/json", - title=self.collection_id, - )) - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) + item.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link(rel="parent", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link( + rel="collection", + target=osc_collection_href, + media_type="application/json", + title=self.collection_id, + ) + ) + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) self.logger.info(f"STAC Item built: {item_href}") return item @@ -650,7 +721,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── item.json # STAC Item (whole Zarr) + └── item.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -662,40 +733,46 @@ def build_zarr_stac_catalog_file_dict( f"Building STAC Catalog file dict for collection '{self.collection_id}' " f"at root '{stac_catalog_s3_root}'." ) + root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - - item = self.build_zarr_stac_item(stac_catalog_s3_root) - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", ) catalog.set_self_href(catalog_href) - catalog.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - catalog.add_link(Link( - rel="item", - target=f"./{self.collection_id}/item.json", - media_type="application/json", - title=self.collection_id, - )) + catalog.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + + item_config = self.items_config[0] + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, + ) + ) + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" - item_href = f"{root}/{self.collection_id}/item.json" self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") return { catalog_href: catalog.to_dict(transform_hrefs=False), item_href: item.to_dict(transform_hrefs=False), } - # ------------------------------------------------------------------ # + # --------------------------------------------------------------------- # # PRR (Project Results Repository) style output # - # # - # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # - # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # - # the plain catalog.json/item.json under ``{root}/prr/``. # - # ------------------------------------------------------------------ # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain {collection_id}/items/{item_id}.json under ``{root}/prr/``. # + # --------------------------------------------------------------------- # - def _get_epsg(self) -> int: + @staticmethod + def _get_crs(dataset: xr.Dataset) -> pyproj.CRS: """Best-effort EPSG code for the dataset, defaulting to 4326. Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or @@ -703,17 +780,20 @@ def _get_epsg(self) -> int: WGS 84 (EPSG:4326). """ for var_name in ("spatial_ref", "crs"): - if var_name in self.dataset.variables: - attrs = self.dataset[var_name].attrs - for key in ("spatial_epsg", "epsg", "EPSG"): - if key in attrs: - try: - return int(attrs[key]) - except (TypeError, ValueError): - pass - return 4326 - - def _get_cube_dimensions(self) -> dict[str, dict]: + if var_name in dataset.variables: + attrs = dataset[var_name].attrs + try: + return pyproj.CRS.from_cf(attrs) + except pyproj.exceptions.CRSError: + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return pyproj.CRS.from_epsg(attrs[key]) + except (TypeError, ValueError): + pass + return pyproj.CRS.from_epsg(4326) + + def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:dimensions`` object from the dataset coordinates. Follows the datacube STAC extension: horizontal spatial dimensions are @@ -721,29 +801,29 @@ def _get_cube_dimensions(self) -> dict[str, dict]: dimension, and any remaining index coordinate is emitted as an additional dimension. """ - ds = self.dataset - epsg = self._get_epsg() + crs = self._get_crs(dataset) x_names = {"lon", "longitude", "x"} y_names = {"lat", "latitude", "y"} dimensions: dict[str, dict] = {} - for name, coord in ds.coords.items(): - if name not in ds.dims: + for name, coord in dataset.coords.items(): + if name not in dataset.dims: # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). continue - lname = str(name).lower() + name = str(name) + lname = name.lower() if lname in x_names: dimensions[name] = { "type": "spatial", "axis": "x", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname in y_names: dimensions[name] = { "type": "spatial", "axis": "y", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname == "time": time_min = pd.to_datetime(coord.min().values).to_pydatetime() @@ -765,11 +845,12 @@ def _get_cube_dimensions(self) -> dict[str, dict]: } return dimensions - def _get_cube_variables(self) -> dict[str, dict]: + @staticmethod + def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:variables`` object from the dataset data variables.""" skip = {"crs", "spatial_ref"} variables: dict[str, dict] = {} - for name, var in self.dataset.data_vars.items(): + for name, var in dataset.data_vars.items(): if name in skip: continue entry: dict = { @@ -782,10 +863,10 @@ def _get_cube_variables(self) -> dict[str, dict]: description = var.attrs.get("long_name") or var.attrs.get("description") if description: entry["description"] = description - variables[name] = entry + variables[str(name)] = entry return variables - def build_prr_stac_item(self) -> Item: + def build_prr_stac_item(self, item_config: ItemConfig) -> Item: """Build the single datacube Item for the PRR collection. One Item covers the full spatiotemporal extent of the Zarr store. It @@ -795,23 +876,27 @@ def build_prr_stac_item(self) -> Item: to fill in via ``Collection.add_item`` + ``normalize_hrefs``. """ self.logger.info( - f"Building PRR STAC Item for collection '{self.collection_id}'." + f"Building PRR STAC Item '{item_config.item_id}' " + f"for collection '{self.collection_id}'." ) - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + dataset = open_dataset(item_config.dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -823,7 +908,7 @@ def build_prr_stac_item(self) -> Item: now_iso = datetime.now(timezone.utc).isoformat() item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, @@ -833,49 +918,63 @@ def build_prr_stac_item(self) -> Item: "description": general_metadata.get("description", ""), "created": now_iso, "updated": now_iso, - "cube:dimensions": self._get_cube_dimensions(), - "cube:variables": self._get_cube_variables(), + "cube:dimensions": self._get_cube_dimensions(dataset), + "cube:variables": self._get_cube_variables(dataset), }, ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) - self.logger.info(f"PRR STAC Item built for '{self.collection_id}'.") + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) + + self.logger.info( + f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." + ) return item def build_prr_collection(self) -> Collection: - """Build the PRR parent Collection with its single datacube Item attached. + """Build the PRR parent Collection with its datacube Item(s) attached. The Collection carries OSC extension fields (``osc:type``, ``osc:status``, ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list and ``processing:datetime`` — aligning with the ESA EarthCODE PRR - endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a - subsequent ``normalize_hrefs`` produces a self-contained tree. + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child, so + it produces a self-contained tree. """ - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + items = [ + self.build_prr_stac_item(item_config) for item_config in self.items_config + ] + spatial_extent = self._union_spatial_extent(items) + temporal_extent = self._union_temporal_extent(items) + dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + variables = self.get_variable_ids(dataset_ref) collection = Collection( id=self.collection_id, - description=general_metadata.get("description", "No description provided."), + description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.collection_id, + title=self.collection_title, ) + collection.stac_version = "1.0.0" osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project @@ -914,16 +1013,14 @@ def build_prr_collection(self) -> Collection: collection.extra_fields["osc:initiative"] = self.osc_initiative project_website = ( - self.osc_project_website - or self.osc_project_url - or self.documentation_link + self.osc_project_website or self.osc_project_url or self.documentation_link ) if project_website: collection.extra_fields["osc:project_website"] = project_website project_description = ( self.osc_project_description or self.description - or general_metadata.get("description") + or "No description provided." ) if project_description: collection.extra_fields["osc:project_description"] = project_description @@ -939,23 +1036,28 @@ def build_prr_collection(self) -> Collection: # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' # with the 'thumbnail' role). if self.thumbnail: - collection.add_asset("thumbnail", Asset( - href=self.thumbnail, - media_type=self._thumbnail_media_type(), - title="Collection Thumbnail", - roles=["thumbnail"], - )) + collection.add_asset( + "thumbnail", + Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + ), + ) if self.documentation_link: collection.add_link( Link(rel="via", target=self.documentation_link, title="Documentation") ) if self.visualisation_link: - collection.add_link(Link( - rel="visualisation", - target=self.visualisation_link, - title="Dataset visualisation", - )) + collection.add_link( + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) + ) try: osc_extension.validate_extension() @@ -963,7 +1065,8 @@ def build_prr_collection(self) -> Collection: raise ValueError(f"OSC Extension validation failed: {e}") self._warn_missing_prr_fields(collection, variables) - collection.add_item(self.build_prr_stac_item()) + for item in items: + collection.add_item(item) return collection def _thumbnail_media_type(self) -> str: @@ -1013,9 +1116,11 @@ def save_prr_collection(self, output_dir: str) -> str: ready to inspect or submit to the ESA EarthCODE PRR endpoint:: {output_dir}/ - ├── collection.json # STAC Collection (root) └── {collection_id}/ - └── {collection_id}.json # datacube Item (whole Zarr) + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) Zarr asset hrefs remain absolute (``s3://…``) since that is where the data lives. @@ -1031,22 +1136,34 @@ def save_prr_collection(self, output_dir: str) -> str: f"'{output_dir}'." ) collection = self.build_prr_collection() - collection.normalize_hrefs(output_dir) + + # Set absolute HREFs for writing. + collection_dir = f"{output_dir}/{self.collection_id}" + items_dir = f"{collection_dir}/items" + collection.set_self_href(f"{collection_dir}/collection.json") + for item in collection.get_items(): + item.set_self_href(f"{items_dir}/{item.id}.json") + + # Write the collection and its children. collection.save(catalog_type=CatalogType.SELF_CONTAINED) self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir - def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: + def build_dataset_stac_collection( + self, mode: str, stac_catalog_s3_root: str | None = None + ) -> Collection: """Build an OSC STAC Collection for the dataset. Returns: A pystac.Collection object. """ try: - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + assert len(self.items_config) == 1 + dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + variables = self.get_variable_ids(dataset) + general_metadata = self._get_general_metadata(dataset) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") @@ -1093,7 +1210,11 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) if self.visualisation_link: collection.add_link( - Link(rel="visualisation", target=self.visualisation_link, title="Dataset visualisation") + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) ) collection.add_link( Link( @@ -1146,7 +1267,7 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) ) - if mode in "all": + if mode == "all": collection.add_link( Link( rel="related", @@ -1168,19 +1289,23 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N catalog_https = self._s3_to_https(catalog_s3) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" - + catalog_https[len("https://"):] + + catalog_https[len("https://") :] + ) + collection.add_link( + Link( + rel="via", + target=stac_browser_href, + title="Access", + ) + ) + collection.add_link( + Link( + rel="child", + target=catalog_https, + media_type="application/json", + title="Items", + ) ) - collection.add_link(Link( - rel="via", - target=stac_browser_href, - title="Access", - )) - collection.add_link(Link( - rel="child", - target=catalog_https, - media_type="application/json", - title="Items", - )) # Validate OSC extension fields try: diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 9452b81..a27a786 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -1,6 +1,5 @@ import logging import os -from typing import Optional import xarray as xr from xcube.core.store import new_data_store @@ -25,8 +24,8 @@ def serialize(obj): def open_dataset( dataset_id: str, root: str = "deep-esdl-public", - storage_configs: Optional[list[dict]] = None, - logger: Optional[logging.Logger] = None, + storage_configs: list[dict] | None = None, + logger: logging.Logger | None = None, ) -> xr.Dataset: """Open an xarray dataset from a specified store. @@ -63,10 +62,15 @@ def open_dataset( "root": os.environ.get("S3_USER_STORAGE_BUCKET", root), "storage_options": { "anon": False, - **({ - "key": os.environ["S3_USER_STORAGE_KEY"], - "secret": os.environ["S3_USER_STORAGE_SECRET"], - } if os.environ.get("S3_USER_STORAGE_KEY") and os.environ.get("S3_USER_STORAGE_SECRET") else {}), + **( + { + "key": os.environ["S3_USER_STORAGE_KEY"], + "secret": os.environ["S3_USER_STORAGE_SECRET"], + } + if os.environ.get("S3_USER_STORAGE_KEY") + and os.environ.get("S3_USER_STORAGE_SECRET") + else {} + ), }, }, }, diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 4a46fb9..9d80152 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from urllib.parse import quote, urlencode, urlparse from xrlint.util.constructible import MappingConstructible @@ -72,13 +72,13 @@ def __init__( description: str, osc_project: str, jupyter_kernel_info: JupyterKernelInfo = None, - osc_workflow: str = None, - updated: str = None, - contacts: list[Contact] = None, - themes: list[Theme] = None, + osc_workflow: str | None = None, + updated: str | None = None, + contacts: list[Contact] | None = None, + themes: list[Theme] | None = None, keywords: list[str] | None = None, formats: list[dict] | None = None, - license: str = None, + license: str | None = None, ): self.created = created self.updated = updated @@ -110,7 +110,7 @@ def to_dict(self, value_name: str | None = None) -> dict[str, JsonValue]: class LinksBuilder: - def __init__(self, themes: list[str], jupyter_kernel_info: dict[str]): + def __init__(self, themes: list[str], jupyter_kernel_info: dict[str, Any]): self.themes = themes self.jupyter_kernel_info = jupyter_kernel_info self.theme_links = [] @@ -152,7 +152,7 @@ def build_child_link_to_related_experiment( } ] - def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, Any]]: + def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> list[dict[str, Any]]: return [ { "rel": "application", @@ -171,7 +171,7 @@ def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, An ] @staticmethod - def _parse_github_notebook_url(url: str) -> Tuple[str, str, str, str]: + def _parse_github_notebook_url(url: str) -> tuple[str, str, str, str]: """ Returns (repo_url, repo_name, branch, file_path_in_repo) from a GitHub URL. @@ -233,7 +233,7 @@ def make_related_link_for_opening_jnb_from_github( jupyter_notebook_url: str, title: str = "Open notebook on the DeepESDL platform", branch_override: str | None = None, - ) -> dict[str, str]: + ) -> list[dict[str, str]]: return [ { "rel": "related", @@ -255,11 +255,13 @@ def __init__( jupyter_notebook_url: str, properties: RecordProperties, links: list[dict], - linkTemplates: list = [], - conformsTo: list[str] = None, - geometry: Optional[Any] = None, - themes: Optional[Any] = None, + linkTemplates: list | None = None, + conformsTo: list[str] | None = None, + geometry: Any | None = None, + themes: Any | None = None, ): + if linkTemplates is None: + linkTemplates = [] if conformsTo is None: conformsTo = [ OGC_API_RECORD_SPEC, @@ -331,8 +333,8 @@ def __init__( properties: RecordProperties, links: list[dict], linkTemplates=None, - conformsTo: list[str] = None, - geometry: Optional[Any] = None, + conformsTo: list[str] | None = None, + geometry: Any | None = None, ): if linkTemplates is None: linkTemplates = [] diff --git a/deep_code/utils/ogc_record_generator.py b/deep_code/utils/ogc_record_generator.py index 83bd08a..e300a60 100644 --- a/deep_code/utils/ogc_record_generator.py +++ b/deep_code/utils/ogc_record_generator.py @@ -16,26 +16,24 @@ class OSCWorkflowOGCApiRecordGenerator: - """Generates OGC API record for a workflow - """ + """Generates OGC API record for a workflow""" @staticmethod def build_contact_objects(contacts_list: list[dict]) -> list[Contact]: """Build a list of Contact objects from a list of contact dictionaries. - Uses the inherited MappingConstructible logic to parse each dict. + Uses the inherited MappingConstructible logic to parse each dict. - Args: - contacts_list: A list of dictionaries, each containing contact information. + Args: + contacts_list: A list of dictionaries, each containing contact information. - Returns: - A list of Contact instances. - """ + Returns: + A list of Contact instances. + """ return [Contact.from_value(cdict) for cdict in contacts_list] @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) diff --git a/docs/cli.md b/docs/cli.md index a269479..6c5c89b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -61,16 +61,20 @@ needs no GitHub credentials or S3 write access (it only reads the Zarr store). I produces a self-contained `Collection → Item → Assets` tree: ``` -prr// -├── collection.json # STAC Collection (root, relative links) -└── / - └── .json # datacube Item covering the whole Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. - The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions and the PRR-mandatory fields. +- `deep-code publish` still publishes one dataset/item at a time; the multi-item + generator support is exposed first through the lower-level API and the PRR helper. The output conforms to the [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) @@ -80,5 +84,4 @@ still runs but logs a warning listing what is needed for full conformance. See Options: -- `--output-dir/-o`: directory to write the tree into. Defaults to `prr_output_dir` - from the config, then `prr/`. +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr`. diff --git a/docs/configuration.md b/docs/configuration.md index 881c572..cd4020d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,7 @@ deep-code generate-config # writes to current directory deep-code generate-config -o ./configs # custom output folder ``` -This creates `dataset_config.yaml` and `workflow_config.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). +This creates `dataset.yaml` and `workflow.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). The sections below document every field in those templates. @@ -16,10 +16,14 @@ The sections below document every field in those templates. ## Dataset config (YAML) ```yaml # Required -dataset_id: your-dataset.zarr collection_id: your-collection # no spaces — use hyphens license_type: CC-BY-4.0 stac_catalog_s3_root: s3://bucket/stac/your-collection/ +items_config: + - dataset_id: your-dataset.zarr + item_id: your-item # no spaces — use hyphens +osc_project: osc-project-name +osc_project_url: osc-project-url # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -27,8 +31,7 @@ osc_region: global dataset_status: completed # ongoing | completed | planned (default: ongoing) documentation_link: https://example.com/docs visualisation_link: https://example.com/viewer # URL to a visualisation of the dataset -osc_project: deep-earth-system-data-lab # defaults to deep-earth-system-data-lab -access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public/{dataset_id} +access_link_root: s3://bucket/ # defaults to s3://deep-esdl-public # CF parameter overrides (list of {name, units, ...} dicts) cf_parameter: @@ -48,21 +51,23 @@ prr_output_dir: ./prr/your-collection ### Field reference -| Field | Required | Description | -|---|---|---| -| `dataset_id` | Yes | Zarr store identifier (used to open the dataset). | -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Dataset-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| Field | Required | Description | +|------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `osc_project` | Yes | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). | +| `osc_project_url` | Yes | OSC project url used to link to project. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | > The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). @@ -93,7 +98,8 @@ prr_output_dir: ./prr/your-collection s3://bucket/stac/your-collection/ ├── catalog.json # STAC Catalog (root) └── your-collection/ - └── item.json # STAC Item covering the full Zarr store + └── items/ + └── your-item.json # STAC Item covering the full Zarr store ``` The item has two assets: @@ -116,10 +122,12 @@ then the boto3 default chain (IAM role, `~/.aws/credentials`). self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): ``` -prr/your-collection/ -├── collection.json # STAC Collection (root, relative links) -└── your-collection/ - └── your-collection.json # datacube Item covering the full Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, @@ -185,4 +193,4 @@ links: | `contact` | No | List of contact objects with `name`, `organization`, and `links`. | | `links` | No | Additional OGC API record links (e.g. `related`, `describedby`). | -More templates and examples live in `dataset_config.yaml`, `workflow_config.yaml`, and `example-config/`. +More templates and examples live in `dataset.yaml`, `workflow.yaml`, and `example-config/`. diff --git a/docs/examples.md b/docs/examples.md index bf3a596..c1eac29 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,5 +1,5 @@ # Examples -- Templates: `dataset_config.yaml`, `workflow_config.yaml` +- Templates: `dataset.yaml`, `workflow.yaml` - Example configs: `examples/example-config/` - Notebooks on publishing: `examples/notebooks` diff --git a/docs/python-api.md b/docs/python-api.md index aa9d0ef..2203e93 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -71,14 +71,14 @@ publisher.publish(write_to_file=False, mode="dataset") over individual artifacts. ```python -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator generator = OscDatasetStacGenerator( - dataset_id="my-dataset.zarr", collection_id="my-collection", workflow_id="my-workflow", workflow_title="My Workflow", license_type="CC-BY-4.0", + items_config=[ItemConfig(dataset_id="my-dataset.zarr", item_id="my-item")], osc_themes=["cryosphere"], osc_region="Global", osc_status="completed", @@ -134,7 +134,7 @@ tree as local files. The high-level helper reads the same dataset config as the from deep_code.tools.prr import generate_prr_collection out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") -# ./prr/collection.json + ./prr//.json +# ./prr/collection.json + ./prr//items/.json ``` Or drive the generator directly: diff --git a/examples/notebooks/publish_to_EarthCODE.ipynb b/examples/notebooks/publish_to_EarthCODE.ipynb index eb8d520..2fec356 100644 --- a/examples/notebooks/publish_to_EarthCODE.ipynb +++ b/examples/notebooks/publish_to_EarthCODE.ipynb @@ -234,11 +234,11 @@ "team_store = new_data_store(\n", " \"s3\", \n", " root=S3_USER_STORAGE_BUCKET, \n", - " storage_options=dict(\n", - " anon=False, \n", - " key=S3_USER_STORAGE_KEY, \n", - " secret=S3_USER_STORAGE_SECRET\n", - " )\n", + " storage_options={\n", + " 'anon': False, \n", + " 'key': S3_USER_STORAGE_KEY, \n", + " 'secret': S3_USER_STORAGE_SECRET\n", + " }\n", ")" ] },