diff --git a/.gitignore b/.gitignore index b97d228..c317627 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ guide/product.yaml earthcode/generators/project_generator.py guide/workflow.yaml guide/experiment.yaml +docs/examples/downloaded_data/ \ No newline at end of file diff --git a/README.md b/README.md index 10ad536..705fc6f 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,6 @@ pip install earthcode ### Requirements The *minimum* required Python version if you would like to install the library locally is 3.12 -### For Developers: - -### Local install -1. `git clone https://github.com/ESA-EarthCODE/earthcode-library.git` -2. Install pixi - https://pixi.sh/dev/installation/ -3. `cd earthcode-library` -4. `pixi install` -5. `pixi run jupyter lab` - -You can run tests through `pixi run pytest`. If running on Windows use ` pixi run pytest --basetemp=C:\t` to avoid long path errors, since some of the project names are >260 chars. ## 2. Quick start ```bash @@ -39,9 +29,8 @@ search("global chlorophyll dataset", variable=chlorophyll.id, type="products", m ``` ## 3. How to contribute to Open Science Catalogue?  -With `earthcode` library creating new entries in the catalogue is semi-automatic. Please find step by step instructions in the `guide/` folder -Start with the `guide/0.Prerequisites - local.ipynb` notebook if you are running the examples locally. -Alternatively, start with `guide/0.Prerequisites-EarthCODE-Workspaces.ipynb` if you are using the [EarthCODE Workspace](https://workspace.earthcode.eox.at/). +With `earthcode` library creating new entries in the catalogue is semi-automatic. Please find step by step instructions in the `docs/` folder. +Start by going through the `docs/contributing_to_the_osc` document. ## 4. Search through Open Science Catalogue In the `examples/` folder you can find notebooks that show: @@ -51,10 +40,23 @@ In the `examples/` folder you can find notebooks that show: We are ready to assist you in case you have any questions/found a bug or mistake, please use GitHub Issues to open a ticket! Alternatively contact the team via e-mail: earthcode@esa.int -If you would like to request a specific feature, or any question to the community of Earthcode and library developers, please use the [EarthCODE discourse forum](https://discourse-earthcode.eox.at/) +If you would like to request a specific feature, or any question to the community of Earthcode and library developers, please use the [EarthCODE discourse Technical Support channel](https://discourse-earthcode.eox.at/c/technical-support/8) + +## Development -## How to contribute to `earthcode` library development? +If you want to contribute a feature to the library you are welcome to contribute via pull request. +First, fork this repository; then make your suggested change and open a PR against this repo. Make sure to add tests for any new functionality. + +### Local install +1. `git clone https://github.com/ESA-EarthCODE/earthcode-library.git` +2. Install pixi - https://pixi.sh/dev/installation/ +3. `cd earthcode-library` +4. `pixi install` +5. `pixi run jupyter lab` + +You can run tests through `pixi run pytest`. If running on Windows use ` pixi run pytest --basetemp=C:\t` to avoid long path errors, since some of the project names are >260 chars. +## To publish a realease Edit pyproject.toml to 1.1.4 and run - `pixi lock` diff --git a/_toc.yml b/_toc.yml index 90efcef..8221cad 100644 --- a/_toc.yml +++ b/_toc.yml @@ -1,19 +1,19 @@ format: jb-book -root: README +root: docs/contributing_to_the_osc.md parts: - - caption: Guide + - caption: Tutorials chapters: - - file: guide/0.Prerequisites-EarthCODE-Workspaces - - file: guide/0.Prerequisites-local - - file: guide/1.Project - - file: guide/2.0.Product - - file: guide/2.1.Product_files_PRR - - file: guide/2.1.Product_files_self_hosted - - file: guide/3.Workflow - - file: guide/4.Experiment - - file: guide/5.Templates - - caption: Examples + - file: docs/tutorials/earthcode_data_discovery + - file: docs/tutorials/end_to_end_subglacial_lakes + - file: docs/tutorials/cog_file_metadata_creation + - file: docs/tutorials/zarr_file_metadata + - file: docs/tutorials/parquet_file_metadata + - caption: Guides chapters: - - file: examples/earthcode_data_discovery - - file: examples/contribute_via_pr_osc - - file: examples/contribute_via_osc_editor + - file: docs/guides/0.Prerequisites-EarthCODE-Workspaces + - file: docs/guides/0.Prerequisites-local + - file: docs/guides/1.Project + - file: docs/guides/2.0.Product + - file: docs/guides/3.Workflow + - file: docs/guides/4.Experiment + - file: docs/guides/5.Templates diff --git a/docs/contributing_to_the_osc.md b/docs/contributing_to_the_osc.md new file mode 100644 index 0000000..1a02133 --- /dev/null +++ b/docs/contributing_to_the_osc.md @@ -0,0 +1,159 @@ +# Contributing to the Open Science Catalogue + +## 1. Introduction + +This document explains how to contribute scientific resources (datasets, workflows, experiments, documentation) and their metadata to the Open Science Catalog (OSC) using the `earthcode-library`. + +The `earthcode-library` is a Python-based toolkit that helps users create, validate, and manage OSC metadata following the catalogue requirements and standards. + +This guide focuses on the end-to-end process and the required steps to contribute: + +* Getting familiar with the main concepts of the Open Science Catalog and the information required to describe your research outcomes. +* Preparing the necessary metadata and associated data assets. +* Generating metadata using the earthcode-library in a format compatible with the Open Science Catalog. +* Submitting your contribution for review and publication. + +Additionally, it describes the available tutorials and how-to-guides. + +**Once you understand the contribution process, follow these steps:** + +1. **Set up your working environment:** + * Create a local environment on your computer or use a remote/cloud workspace. + * Install the EarthCODE Library and any required dependencies. +2. **Create metadata to describe your scientific outcomes:** + * Develop **your own Python script or Jupyter Notebook** that generates the metadata. + * Use the earthcode-library to facilitate the process and validate the metadata against the Open Science Catalogue requirements. +3. **Generate file-level metadata** to ensure data long-term accessibility and findability. +4. **Submit your contribution:** + * Open a Pull Request (PR) against the Open Science Catalog repository and start the review process. + +## 2. What is the Open Science Catalog? + +The Open Science Catalog (OSC) indexes Earth Observation research outputs from ESA-funded activities. It helps users find scientific products and datasets, workflows, and experiments. It also links the scientific outcomes to the geophysical variables that describe them and related EO-missions. All products and workflows are categorised by themes corresponding to the environmental domain to facilitate product discoverability. + +The OSC is built around STAC, the Spatio-Temporal Asset Catalog standard. STAC provides a common way to describe geospatial and temporal data, the assets that belong to it, and the links between related resources. + +The main OSC concepts are: + +* **`Project`:** the ESA-funded Earth Observation Programme activity or research. A project records the official title, description, time span, spatial extent, environmental domain (theme), contact to ESA Technical Officer, list of consortium member(s), links to external project website, and product(s) generated by project. +* **`Product`:** the scientific output that users should be able to discover, understand, access, cite, and reuse. A product should include a clear description, licensing information, contributing EO missions, geophysical variables, spatial and temporal coverage, provenance, and access information. +* **`Assets`:** the actual files forming a product. Each file, archive, scene, or time-specific data object should be described with file-level metadata. +* **`Workflow`:** the reusable scientific code, processing logic, or method used to generate a product. In the Open Science Catalog, workflows are linked to projects, provided with a specific license, version, and the execution environment (usually stored in a GitHub repo or stored in an application package). +* **`Experiment`:** a specific execution or implementation of a workflow that provides a complete context to generate a product. It provides information on input data and parameters, configuration settings, and the runtime environment used to create the output. Each experiment is linked to generated products, the workflow that was used, and the project under which it was developed. +* **`Variables`:** controlled terms that describe the geophysical, climate, or environmental quantities represented by a product. Variables are selected from recognised community standards, including the WMO OSCAR and GCMD keyword databases, to ensure consistent discovery and interoperability across datasets. +* **`EO missions`:** controlled terms that are used to identify the Earth Observation satellite mission(s) or observing systems used to generate a product. Products based on in-situ data or numerical models should use the relevant source category, also stored under the category of eo-missions concept. +* **`Themes`:** terms used to gather products, projects, workflows, and experiments under a specific environmental domain. Themes are used to facilitate the discovery of these concepts. + +Each **project, product, workflow, and experiment** must be associated with a **clear and valid license type** to define how the resource can be accessed, used, shared, and reused. Licensing information is required for publication in the Open Science Catalogue, and missing or ambiguous licenses may prevent the resource from being published. + +The key principle of the Open Science Catalogue (OSC) is that all published records are interconnected. This is implemented to enable traceability and transparency between ESA-funded EO projects and their scientific outputs. A **product** links to its parent **project**, **file-level metadata** links individual assets to the product, a **workflow** documents the method used to generate the results, and an **experiment** captures a specific execution of that method. Together these concepts make the output discoverable, reusable, and reviewable. + +## 3. Contributing to the Open Science Catalog + +To publish your data in the Open Science Catalogue, you need: + +1. **Host your data in a Long-Term Repository** + Your data should be hosted in a long-term repository in a format suitable for cloud-based object storage and streaming. Data is easiest to publish when it is already stored at stable URLs and is in a cloud-optimised format. The preferred formats are Zarr, COG, and Parquet. + +| Format | Best for | Why it matters in object storage | +| --- | --- | --- | +| Zarr | Multidimensional arrays such as time/depth/y/x cubes | Supports chunked access and analysis to load only the required portions of the full dataset. | +| Cloud Optimized GeoTIFF (COG) | Single rasters or raster stacks | Internal tiling and overviews enable efficient range reads and visualisation. | +| GeoParquet | Vector geometries and tabular point data | Supports efficient filtering and querying through columnar storage and embedded geospatial metadata. | + +> See the examples for conversion patterns. +> +> **If your data is not yet in a suitable format, the EarthCODE team can help you decide what conversion or hosting path is appropriate. If you are at this stage, email the EarthCODE team at earth-code@esa.int** +> +> If your data is NOT hosted on object storage, the EarthCODE team can help move your data to ESA object storage. You still need to complete all the steps below and open the PR to provide enough information to support you. + +2. **Metadata** + Provide the metadata describing your project, products, workflows, experiments, and associated data assets, following the requirements outlined in **Section 2**. + +3. **Pull Request submission** + Submit the Pull Request (PR) to the `open-science-catalog-metadata` repository with the information from Step 1 and 2. In the PR you can also add any additional information supporting the publication and review process. + +## 4. Using the EarthCODE Library + +The EarthCODE Library helps users create, save, validate, and search Open Science Catalogue (OSC) metadata. It does not replace scientific judgement: contributors still need to provide accurate descriptions, licenses, extents, variables, missions, workflow links, experiment context, and file-level metadata. + +Use the library and follow the contribution path below. + +1. **Set up your working environment.** + You can contribute to OSC by using the EarthCODE Library from a local environment or from an EarthCODE Workspace - https://workspace.earthcode.eox.at/. + * Use the local setup how-to guide if you want to install the library, clone your OSC metadata fork, run notebooks locally, and open a pull request from your own machine. + * Use the EarthCODE workspace setup how-to guide if you want to work in the workspace with JupyterLab and the integrated platform tooling. + +2. **Create metadata to describe your scientific outcomes.** + **Create a new Jupyter Notebook** or Python script to add project, product, workflow, and experiment metadata to your `open-science-catalog-metadata` fork. + We recommend creating records in the following order: + 1) **Project:** provides top-level context for the contribution. + 2) **Product(s):** to describe the dataset(s) which are linked to a Project. + 3) **Workflow:** code or processing logic used to generate a Product. + 4) **Experiment:** specific workflow execution used to generate a specific Product. + +3. **Generate File-Level Metadata.** + In addition to product metadata, you need to generate STAC Item metadata for each file belonging to a product (specified in Step 2). + Product metadata describes a dataset as a whole. STAC Item metadata describes the actual dataset files. This is often the most time-consuming part of a contribution because each asset needs enough information to be usable: stable URL, title or meaningful id, description, temporal extent, spatial footprint and bounding box, file format and MIME type, license, and any useful extra fields (which can be customised). + +### Special Requests + +**If your data is NOT yet hosted online** (or does not have a stable URL): +1. You still need to complete the project, product, workflow, experiment and produce file-level metadata. +2. Use the file name as a placeholder in the file URL field. (Provide an additional comment in your Pull Request describing this issue). The EarthCODE team will change it once the data is hosted online. +3. Open the Pull Request (PR) against the open-science-metadata repository. +4. Email the earthcode team at earth-code@esa.int. + +**For STAC collections with more than 600 STAC Items:** +1. Create a self-contained STAC Collection and add the Items to it. +2. Open the Pull Request against the open-science-metadata repository. +3. Package (zip) the self-contained STAC Collection and send it to the EarthCODE team (earth-code@esa.int). +4. The EarthCODE team will assist in hosting the Collection and update the links. + +4. **Submit the Pull Request (PR).** + When the metadata is generated and validation passes: + * Commit your changes to your fork of the open-science-catalog-metadata repository. + * Open a Pull Request against the main repository. + +The pull request should contain only the intended OSC metadata changes. It should include a clear title and description so the EarthCODE Data Steward team can understand what is being added or changed. + +After the Pull Request is opened, automatic checks validate the metadata. The EarthCODE Data Steward team reviews the contribution and may ask for corrections before merging it into the published catalogue. + +### Tutorials + +The following tutorials demonstrate with code different aspects of the Open Science Catalogue contribution process: + +* A complete end-to-end example showing how you can add a project, product, workflow, experiment, and product-file metadata to the Open Science Catalogue - `docs/tutorials/end_to_end_subglacial_lakes.ipynb`. +* Search and discovery: a tutorial on how to search existing OSC data so contributors can reuse existing projects, variables, EO missions, themes, and examples where appropriate - `docs/tutorials/earthcode_data_discovery.ipynb`. +* How to generate STAC Item metadata for the main supported file formats: + * Generate STAC Item metadata for Zarr data - `docs/tutorials/zarr_file_metadata.ipynb`. + * Generate STAC Item metadata for COG data - `docs/tutorials/cog_file_metadata.ipynb`. + * Generate STAC Item metadata for Parquet data - `docs/tutorials/parquet_file_metadata.ipynb`. + +### How-to guides + +How-to guides show **how you can complete a specific task with your own data.** You can start working through them in order if you like. You will have to combine different aspects of various how-to-guides in order to make a contribution. + +The how-to guides include: + +* `docs/guides/0.Prerequisites-EarthCODE-Workspaces.ipynb` - how to setup the environment to run the rest of the notebooks and commit the results locally. +* `docs/guides/Prerequisites-EarthCODE-Workspace.ipynb` - how to setup the environment to run the rest of the notebooks and commit the results, using the EarthCODE workspace. +* `docs/guides/1.Project.ipynb` - create and validate **project** metadata. +* `docs/guides/2.Product.ipynb` - create and validate **product** metadata. +* `docs/guides/3.Workflow.ipynb` - create and validate **workflow** metadata. +* `docs/guides/4.Experiment.ipynb` - create and validate **experiment** metadata. +* `docs/guides/5.Templates.ipynb` - if you prefer the **YAML-template workflow.** + +### Next steps + +Start contributing by: + +1. Creating a local or remote environment. +2. Creating your own script or Jupyter notebook to generate the metadata for your project, products, and data files. +3. Opening a PR against the Open Science Catalog to start the review process, OR email the earthcode team for support. + +### Looking for support? + +We are ready to assist you in case you have any questions, please use GitHub Issues or contact the team via e-mail: earth-code@esa.int. + +If you would like to request a specific feature, or have any question for the community of EarthCODE and library developers, please use the EarthCODE discourse forum. \ No newline at end of file diff --git a/guide/0.Prerequisites-EarthCODE-Workspaces.ipynb b/docs/guides/0.Prerequisites-EarthCODE-Workspaces.ipynb similarity index 100% rename from guide/0.Prerequisites-EarthCODE-Workspaces.ipynb rename to docs/guides/0.Prerequisites-EarthCODE-Workspaces.ipynb diff --git a/guide/0.Prerequisites-local.ipynb b/docs/guides/0.Prerequisites-local.ipynb similarity index 98% rename from guide/0.Prerequisites-local.ipynb rename to docs/guides/0.Prerequisites-local.ipynb index da97524..1e0a590 100644 --- a/guide/0.Prerequisites-local.ipynb +++ b/docs/guides/0.Prerequisites-local.ipynb @@ -7,14 +7,18 @@ "source": [ "# Prerequisites - local \n", "\n", - "## Short introduction to EarthCODE\n", - "ESA has developed [EarthCODE](https://earthcode.esa.int/), a Collaborative Open Development Environment for Earth System Science. The scope of EarthCODE is to enable adoption of [FAIR](https://esa-earthcode.github.io/documentation/Community%20and%20Best%20Practices/FAIR%20and%20Open%20Science%20Best%20Practices/) and Open Science Principles throughout ESA-funded Earth Science activities and supporting the ESA Science Clusters, to deliver long-term persistence of data, code and documentation, aiding reproducibility, reuse and consumption of research outputs by a wider community. EarthCODE integrates 1) the [Open Science Catalogue](https://opensciencedata.esa.int/), making it effortless to discover novel research outcomes from ESA Science Clusters activities; 2) access to commercial platform services; and 3) a wealth of community resources and tools that help maintain data and research quality.
\n", + "\n", "\n", "In this guide we provide steps on how to expoit resources provided in this first compontent related to data cataloguing and sharing.
\n", "\n", + "\n", + "## Short introduction to EarthCODE\n", + "ESA has developed [EarthCODE](https://earthcode.esa.int/), a Collaborative Open Development Environment for Earth System Science. The scope of EarthCODE is to enable adoption of [FAIR](https://esa-earthcode.github.io/documentation/Community%20and%20Best%20Practices/FAIR%20and%20Open%20Science%20Best%20Practices/) and Open Science Principles throughout ESA-funded Earth Science activities and supporting the ESA Science Clusters, to deliver long-term persistence of data, code and documentation, aiding reproducibility, reuse and consumption of research outputs by a wider community. EarthCODE integrates 1) the [Open Science Catalogue](https://opensciencedata.esa.int/), making it effortless to discover novel research outcomes from ESA Science Clusters activities; 2) access to commercial platform services; and 3) a wealth of community resources and tools that help maintain data and research quality.
\n", + "\n", "The Open Science Catalogue is built on the [Spatio Temporal Asset Catalog (STAC)](https://stacspec.org/en), which is a standard for describing geospatial data. Therefore new entries must conform to its specification.\n", "We've broken the process into six steps, contained within six notebooks. You can run the individual notebooks and commit the results via pull request to the [open science catalog metadata repository](https://github.com/ESA-EarthCODE/open-science-catalog-metadata/tree/main).
\n", - "Below you can find instructions about setting up the environment, running the notebooks and creating a pull request.\n", + "\n", + "###\n", "\n", "## 0.1 Install the earthcode library\n", "\n", diff --git a/guide/1.Project.ipynb b/docs/guides/1.Project.ipynb similarity index 96% rename from guide/1.Project.ipynb rename to docs/guides/1.Project.ipynb index 6948312..822b43f 100644 --- a/guide/1.Project.ipynb +++ b/docs/guides/1.Project.ipynb @@ -98,7 +98,8 @@ "to_name = \"\"\n", "to_email = \"\"\n", "\n", - "# List the consortium members in a tuple with format (name, contact_email), for example - ('University A', \"contact@universitya.fr\")\n", + "# List the consortium members in a tuple with format (name, contact_email), for example - [('University A', \"contact@universitya.fr\"), ('University B', \"contact@universityb.fr\")]\n", + "# if you dont want to add an email, add an empty string in the second tuple position - [('University A', \"\"), ('University B', \"\")]\n", "consortium_members = [(\"\", \"\")]" ] }, diff --git a/guide/2.0.Product.ipynb b/docs/guides/2.0.Product.ipynb similarity index 99% rename from guide/2.0.Product.ipynb rename to docs/guides/2.0.Product.ipynb index 737694e..3059184 100644 --- a/guide/2.0.Product.ipynb +++ b/docs/guides/2.0.Product.ipynb @@ -172,6 +172,7 @@ " project_title=project_title,\n", " product_parameters=product_parameters,\n", " product_doi=product_doi,\n", + " access_link=f'https://opensciencedata.esa.int/products/{product_id}/collection',\n", " license_link=license_link or None,\n", ")\n", "\n", diff --git a/guide/3.Workflow.ipynb b/docs/guides/3.Workflow.ipynb similarity index 93% rename from guide/3.Workflow.ipynb rename to docs/guides/3.Workflow.ipynb index bcdc7a6..2b17179 100644 --- a/guide/3.Workflow.ipynb +++ b/docs/guides/3.Workflow.ipynb @@ -61,7 +61,8 @@ "# If you have multiple licenses, you can pick 'various'\n", "workflow_license = \"CC-BY-4.0\"\n", "\n", - "# what DATA the workflow takes as input and output, i.e. GeoTIFF, Netcdf\n", + "# what DATA the workflow takes as either input and output, i.e. GeoTIFF, Netcdf. \n", + "# These could be different values\n", "workflow_formats = [\"netcdf64\"]\n", "\n", "# Define which project the workflow is associated with\n", @@ -76,18 +77,14 @@ "# See the list here: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/blob/main/themes/catalog.json\n", "workflow_themes = [\"\"]\n", "\n", - "# List the contacts in a tuple with format (name, contact_email), for example - ('Magellium', \"contact@magellium.fr\")\n", - "workflow_contracts_info = [(\"\", \"\")]\n", - "\n", "# Define the access to the repository where the workflow can be discovered. Provide an active URL below:\n", - "codeurl = \"\"\n", + "workflow_url = \"\"\n", "\n", "# Optional workflow record fields\n", "workflow_doi = None\n", - "workflow_w = -180.0\n", - "workflow_s = -90.0\n", - "workflow_e = 180.0\n", - "workflow_n = 90.0\n", + "\n", + "\n", + "\n", "workflow_start_year = 2021\n", "workflow_start_month = 1\n", "workflow_start_day = 1\n", @@ -130,11 +127,10 @@ " workflow_keywords=workflow_keywords,\n", " workflow_formats=workflow_formats,\n", " workflow_themes=workflow_themes,\n", - " codeurl=codeurl,\n", + " codeurl=workflow_url,\n", " project_id=project_id,\n", " project_title=project_title,\n", " workflow_doi=workflow_doi,\n", - " workflow_bbox=[[workflow_w, workflow_s, workflow_e, workflow_n]] if include_workflow_bbox else None,\n", " workflow_start_datetime=datetime(workflow_start_year, workflow_start_month, workflow_start_day) if include_workflow_time else None,\n", " workflow_end_datetime=datetime(workflow_end_year, workflow_end_month, workflow_end_day) if include_workflow_time else None,\n", ")\n", diff --git a/guide/4.Experiment.ipynb b/docs/guides/4.Experiment.ipynb similarity index 90% rename from guide/4.Experiment.ipynb rename to docs/guides/4.Experiment.ipynb index 97173f0..a813fd9 100644 --- a/guide/4.Experiment.ipynb +++ b/docs/guides/4.Experiment.ipynb @@ -56,7 +56,7 @@ "experiment_license = \"\"\n", "experiment_keywords = [\"\"]\n", "\n", - "# Define the input output formats that this experiment works with\n", + "# Define the all the output formats that this experiment produces\n", "# i.e. GeoTIFF, Zarr, netCDF, etc\n", "experiment_formats = [\"\"]\n", "\n", @@ -64,10 +64,10 @@ "# - atmosphere, cryosphere, land, magnetosphere-ionosphere, oceans, solid-earth.\n", "experiment_themes = [\"\"]\n", "\n", - "# link to the specification of the input parameters for the experiment\n", + "# link to the specification of the input parameters for the experiment. Ideally this should be yaml file specifying the input parameters to the worklfow\n", "experiment_input_parameters_link = \"\"\n", - "# link to the enviroment in which the experiment was performed\n", - "experiment_enviroment_link = \"\"\n", + "# link to the enviroment in which the experiment was performed. Ideally this should be a yaml/docker file describing the enviroment\n", + "experiment_environment_link = \"\"\n", "\n", "## ID and title of the associated workflow\n", "workflow_id = \"\"\n", @@ -78,18 +78,18 @@ "product_title = \"\"\n", "\n", "# Optional experiment record fields\n", + "\n", + "# List the contacts in a tuple with format (name, contact_email), for example - ('Magellium', \"contact@magellium.fr\")\n", "experiment_contacts = None\n", - "experiment_w = -180.0\n", - "experiment_s = -90.0\n", - "experiment_e = 180.0\n", - "experiment_n = 90.0\n", + "\n", + "# temporal extent of the experiment\n", "experiment_start_year = 2021\n", "experiment_start_month = 1\n", "experiment_start_day = 1\n", "experiment_end_year = 2021\n", "experiment_end_month = 12\n", "experiment_end_day = 31\n", - "include_experiment_bbox = False\n", + "\n", "include_experiment_time = False\n", "\n", "# Local OSC clone root path (assumed one folder above repository root)\n", @@ -111,12 +111,14 @@ { "cell_type": "code", "execution_count": null, - "id": "2c01bf43", + "id": "7a4ade25", "metadata": {}, "outputs": [], "source": [ "catalog_root = Path(catalog_root)\n", "\n", + "\n", + "# create an OSC Experiment Record\n", "experiment_metadata = ExperimentMetadata(\n", " experiment_id=experiment_id,\n", " experiment_title=experiment_title,\n", @@ -126,20 +128,22 @@ " experiment_formats=experiment_formats,\n", " experiment_themes=experiment_themes,\n", " experiment_input_parameters_link=experiment_input_parameters_link,\n", - " experiment_enviroment_link=experiment_enviroment_link,\n", + " experiment_enviroment_link=experiment_environment_link,\n", " workflow_id=workflow_id,\n", " workflow_title=workflow_title,\n", " product_id=product_id,\n", " product_title=product_title,\n", " contacts=experiment_contacts,\n", - " experiment_bbox=[[experiment_w, experiment_s, experiment_e, experiment_n]] if include_experiment_bbox else None,\n", " experiment_start_datetime=datetime(experiment_start_year, experiment_start_month, experiment_start_day) if include_experiment_time else None,\n", " experiment_end_datetime=datetime(experiment_end_year, experiment_end_month, experiment_end_day) if include_experiment_time else None,\n", ")\n", "\n", "experiment_record = create_experiment_record(experiment_metadata)\n", + "\n", + "# save the experiment to the OSC\n", "save_experiment_record_to_osc(experiment_record, catalog_root)\n", "\n", + "# Validate the output\n", "errors, error_files = validate_catalog(catalog_root)\n", "if errors or error_files:\n", " raise AssertionError(f\"Catalog validation failed. errors={len(errors)} files={len(error_files)}\")\n", diff --git a/guide/5.Templates.ipynb b/docs/guides/5.Templates.ipynb similarity index 100% rename from guide/5.Templates.ipynb rename to docs/guides/5.Templates.ipynb diff --git a/docs/tutorials/cog_file_metadata_creation.ipynb b/docs/tutorials/cog_file_metadata_creation.ipynb new file mode 100644 index 0000000..7288d5c --- /dev/null +++ b/docs/tutorials/cog_file_metadata_creation.ipynb @@ -0,0 +1,1103 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e07c81ed", + "metadata": {}, + "source": [ + "# Build STAC Metadata For A COG\n", + "\n", + "This notebook derives spatial and raster metadata from the basal-melt COG and uses it to build a STAC Item. The same pattern applies to cloud-hosted COGs: derive machine-checkable fields from the asset wherever possible, then add human and provenance metadata explicitly." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2d4566d8", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import datetime as dt\n", + "import json\n", + "import pystac\n", + "import rasterio\n", + "from pystac.extensions.projection import ProjectionExtension\n", + "from pystac.extensions.raster import RasterBand, RasterExtension\n", + "from rasterio.warp import transform_bounds\n", + "from shapely.geometry import box, mapping\n", + "\n", + "COG_PATH = \"https://s3.waw4-1.cloudferro.com/EarthCODE/OSCAssets/polar_cube_datasets/basal_melt/basal_melt_map_racmo_firn_air_corrected.tif\"" + ] + }, + { + "cell_type": "markdown", + "id": "36a65cfb", + "metadata": {}, + "source": [ + "## Derive Spatial And Raster Fields\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bfd97f4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EPSG: 3031\n", + "WGS84 bbox: (-180.0, -90.0, 180.0, -50.68637942059157)\n", + "Shape: [6435, 6120]\n", + "Data type: float32, nodata: -3.4028234663852886e+38\n" + ] + } + ], + "source": [ + "with rasterio.open(COG_PATH) as src:\n", + " bounds_native = src.bounds\n", + " bbox_wgs84 = transform_bounds(src.crs, \"EPSG:4326\", *bounds_native, densify_pts=21)\n", + " geometry_wgs84 = mapping(box(*bbox_wgs84))\n", + " epsg = src.crs.to_epsg()\n", + " transform = [src.transform.a, src.transform.b, src.transform.c, src.transform.d, src.transform.e, src.transform.f]\n", + " shape = [src.height, src.width]\n", + " data_type = src.dtypes[0]\n", + " nodata = src.nodata\n", + " resolution = abs(src.res[0])\n", + "\n", + "print(f\"EPSG: {epsg}\")\n", + "print(f\"WGS84 bbox: {bbox_wgs84}\")\n", + "print(f\"Shape: {shape}\")\n", + "print(f\"Data type: {data_type}, nodata: {nodata}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "23e36b7b", + "metadata": {}, + "source": [ + "## Create And Validate The STAC Item\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1e2e37ae", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'type': 'Feature',\n", + " 'stac_version': '1.1.0',\n", + " 'stac_extensions': ['https://stac-extensions.github.io/projection/v2.0.0/schema.json',\n", + " 'https://stac-extensions.github.io/raster/v1.1.0/schema.json'],\n", + " 'id': 'antarctic-ice-shelf-basal-melt-cog',\n", + " 'geometry': {'type': 'Polygon',\n", + " 'coordinates': (((180.0, -90.0),\n", + " (180.0, -50.68637942059157),\n", + " (-180.0, -50.68637942059157),\n", + " (-180.0, -90.0),\n", + " (180.0, -90.0)),)},\n", + " 'bbox': [-180.0, -90.0, 180.0, -50.68637942059157],\n", + " 'properties': {'title': 'Antarctic ice shelf basal melt rate COG',\n", + " 'description': 'Cloud Optimized GeoTIFF example for Antarctic ice shelf basal melt rates.',\n", + " 'license': 'cc-by-4.0',\n", + " 'start_datetime': '1997-01-01T00:00:00Z',\n", + " 'end_datetime': '2021-12-31T23:59:59Z',\n", + " 'proj:code': 'EPSG:3031',\n", + " 'proj:shape': [6435, 6120],\n", + " 'proj:bbox': [-3060277.25, -3217740.25, 3059722.75, 3217259.75],\n", + " 'proj:transform': [1000.0, 0.0, -3060277.25, 0.0, -1000.0, 3217259.75],\n", + " 'datetime': '2021-01-01T00:00:00Z'},\n", + " 'links': [],\n", + " 'assets': {'basal_melt_rate': {'href': 'https://s3.waw4-1.cloudferro.com/EarthCODE/OSCAssets/polar_cube_datasets/basal_melt/basal_melt_map_racmo_firn_air_corrected.tif',\n", + " 'type': 'image/tiff; application=geotiff; profile=cloud-optimized',\n", + " 'title': 'Basal melt rate COG',\n", + " 'raster:bands': [{'nodata': -3.4028234663852886e+38,\n", + " 'data_type': 'float32',\n", + " 'spatial_resolution': 1000.0,\n", + " 'unit': 'm yr-1'}],\n", + " 'roles': ['data']}}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item = pystac.Item(\n", + " id=\"antarctic-ice-shelf-basal-melt-cog\",\n", + " geometry=geometry_wgs84,\n", + " bbox=list(bbox_wgs84),\n", + " datetime=dt.datetime(2021, 1, 1, tzinfo=dt.timezone.utc),\n", + " properties={\n", + " \"title\": \"Antarctic ice shelf basal melt rate COG\",\n", + " \"description\": \"Cloud Optimized GeoTIFF example for Antarctic ice shelf basal melt rates.\",\n", + " \"license\": \"cc-by-4.0\",\n", + " \"start_datetime\": \"1997-01-01T00:00:00Z\",\n", + " \"end_datetime\": \"2021-12-31T23:59:59Z\",\n", + " },\n", + ")\n", + "\n", + "ProjectionExtension.add_to(item)\n", + "proj = ProjectionExtension.ext(item)\n", + "proj.epsg = epsg\n", + "proj.shape = shape\n", + "proj.bbox = list(bounds_native)\n", + "proj.transform = transform\n", + "\n", + "asset = pystac.Asset(\n", + " href=COG_PATH,\n", + " media_type=pystac.MediaType.COG,\n", + " roles=[\"data\"],\n", + " title=\"Basal melt rate COG\",\n", + ")\n", + "RasterExtension.add_to(item)\n", + "item.add_asset(\"basal_melt_rate\", asset)\n", + "\n", + "raster = RasterExtension.ext(item.assets[\"basal_melt_rate\"])\n", + "raster.bands = [\n", + " RasterBand.create(\n", + " nodata=nodata,\n", + " data_type=data_type,\n", + " spatial_resolution=resolution,\n", + " unit=\"m yr-1\",\n", + " )\n", + "]\n", + "\n", + "item.validate()\n", + "item.to_dict()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf902ff9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b913a0f1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f1ae8417", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from earthcode.static import generic_stac_item_to_earthcode_stac_item\n", + "from earthcode.git_add import save_item_to_product_collection\n", + "\n", + "# specify catalog root and OSC collection\n", + "catalog_root = Path('../../../open-science-catalog-metadata/')\n", + "collection = 'sofresh-sea-surface-salinity'" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "347a1528", + "metadata": {}, + "outputs": [], + "source": [ + "# convert to earthcode item with a reference to a specific open science catalog collection\n", + "earthcode_item = generic_stac_item_to_earthcode_stac_item(item, collection, asset_key='data')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "4ce8fbe3", + "metadata": {}, + "outputs": [], + "source": [ + "# save the item to the collection\n", + "save_item_to_product_collection(earthcode_item, collection, catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ce740fd8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([], [])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# validate catalog\n", + "from earthcode.validator import validate_catalog\n", + "validate_catalog(catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "bac6e651", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=antarctic-ice-shelf-basal-melt-cog>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "earthcode_item" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/earthcode_data_discovery.ipynb b/docs/tutorials/earthcode_data_discovery.ipynb similarity index 100% rename from examples/earthcode_data_discovery.ipynb rename to docs/tutorials/earthcode_data_discovery.ipynb diff --git a/docs/tutorials/end_to_end_subglacial_lakes.ipynb b/docs/tutorials/end_to_end_subglacial_lakes.ipynb new file mode 100644 index 0000000..9ae09bf --- /dev/null +++ b/docs/tutorials/end_to_end_subglacial_lakes.ipynb @@ -0,0 +1,4632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "osc-tutorial-intro", + "metadata": {}, + "source": [ + "# Tutorial: create OSC metadata for subglacial lakes\n", + "\n", + "This tutorial shows how to create a Open Science Catalog (OSC) metadata contribution with the EarthCODE Library. Use the tutorial to structure your the script/notebook for your own contribution to the OSC.\n", + "\n", + "You will see an example, how to:\n", + "\n", + "1. Step 1. setup the local enviroment\n", + "2. Step 2. Add metadata\n", + "- a `Project` collection for the ESA research context;\n", + "- a `Product` collection for the discoverable dataset;\n", + "- a `Workflow` record for the code or method;\n", + "- an `Experiment` record linking the created `Workflow` and `Product`;\n", + "3. Step 3. Add STAC Item level medata for each of your files\n", + "- one file-level STAC Item for a Parquet asset;\n", + "4. Step 4. Run validation\n", + "- a final validation report for the local catalog checkout.\n", + "5. Step 5: Open a PR\n", + "\n", + "\n", + "This notebook uses made up IDs so it can be run without changing an existing published record. \n", + "\n", + "> Do not open a pull request from this example. " + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-prereqs", + "metadata": {}, + "source": [ + "## Step 1. Setup a local enviroment\n", + "\n", + "You need an environment with the EarthCODE Library installed and a local fork or clone of `open-science-catalog-metadata`. The file-level metadata step also reads a small public Parquet file from object storage.\n", + "\n", + "\n", + "To see how to setup your enviroment in detail checkout the `guides/0.Prerequisites-local.ipynb` guide.\n", + "\n", + "In summary, you should:\n", + "- Fork the open science catalog repository on github - [https://github.com/ESA-EarthCODE/open-science-catalog-metadata](https://github.com/ESA-EarthCODE/open-science-catalog-metadata)\n", + "- Clone the fork locally\n", + "- Install the earthcode-library\n", + "\n", + "\n", + "\n", + "The save and validation helpers write into a local `open-science-catalog-metadata` instance. Set `catalog_root` to that instance's path before you create any metadata records." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7ee85846", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "from earthcode.metadata_input_definitions import ProductCollectionMetadata\n", + "from earthcode.static import create_product_collection\n", + "from earthcode.git_add import save_product_collection_to_catalog\n", + "\n", + "from earthcode.metadata_input_definitions import ProjectCollectionMetadata\n", + "from earthcode.static import create_project_collection\n", + "from earthcode.git_add import save_project_collection_to_osc\n", + "from earthcode.validator import validate_catalog\n", + "\n", + "from earthcode.search import search\n", + "import geopandas as gpd\n", + "import pystac" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "14778371", + "metadata": {}, + "outputs": [], + "source": [ + "catalog_root = Path(\"../../../open-science-catalog-metadata\").resolve()" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-metadata-overview", + "metadata": {}, + "source": [ + "## Step 2. Create the catalog records\n", + "\n", + "Start with the catalog-level records before adding file-level metadata.\n" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-project-inputs", + "metadata": {}, + "source": [ + "### Create the Project metadata\n", + "\n", + "A Project is the top-level OSC context for an ESA activity. In this step you provide the project ID, title, description, status, license, spatial and temporal coverage, science themes, Technical Officer contact, website links, and consortium members.\n", + "\n", + "For your own contribution, replace the example values with the official project metadata. Keep IDs short, lowercase, and dash-separated.\n", + "\n", + "See the `guides/1.Project.ipynb` guide for more information about the project collection." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "80b702f4", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "# A custom id of the project, it can be related to the title, i.e. - 4datlantic-ohc. Use dash \"-\" symbol to separare words in the id\"\n", + "project_id = \"4d-antarctica-v2\"\n", + "\n", + "# Specify the Title of your project. I.e. - 4DAtlantic-OHC. This should correspond to the title of the project as in the ESA contract.\n", + "project_title = \"4D-ANTARCTICA-v2\"\n", + "\n", + "# A short description of the project:\n", + "project_description = \"Ice sheets are a key component of the Earth system, impacting on global sea level, ocean circulation and bio geochemical processes. Significant quantities of liquid water are being produced and transported at the ice sheet surface, base, and beneath its floating sections, and this water is in turn interacting with the ice sheet itself. Despite the key role that hydrology plays on the ice sheet environment, there is still no global hydrological budget for Antarctica. There is currently a lack of global data on supra: and sub:glacial hydrology, and no systems are in place for continuous monitoring of it or its impact on ice dynamics. The overall aim of 4DAntarctica is to advance our understanding of the Antarctic Ice Sheets supra and sub:glacial hydrology, its evolution, and its role within the broader ice sheet and ocean systems. We designed our programme of work to address the following specific objectives: Creating and consolidating an unprecedented dataset composed of ice:sheet wide hydrology and lithospheric products, Earth Observation datasets, and state of the art ice sheet and hydrology models; Improving our understanding of the physical interaction between electromagnetic radiation, the ice sheet, and liquid water; Developing techniques and algorithms to detect surface and basal melting from satellite observations in conjunction with numerical modelling; Applying these new techniques at local sites and across the continental ice sheet to monitor water dynamics and derive new hydrology datasets; Performing a scientific assessment of Antarctic Ice Sheet hydrology and of its role in the current changes the continent is experiencing; Proposing a future roadmap for enhanced observation of Antarcticas hydrological cycle. To do so, the project will use a large range of Earth Observation missions (e.g. Sentinel 1, Sentinel 2, SMOS, CryoSat 2, GOCE, TanDEM X, AMSR2, Landsat, Icesat 2) coupled with ice:sheet and hydrological models. By the end of this project, the programme of work presented here will lead to a dramatically improved quantification of meltwater in Antarctica, an improved understanding of fluxes across the continent and to the ocean, and an improved understanding of the impact of the hydrological cycle on ice sheets mass balance, its basal environment, and its vulnerability to climate change.\"\n", + "\n", + "# Project status: pick from - ongoing or completed\n", + "project_status = \"completed\"\n", + "\n", + "# Overall license for all related data that will be uploaded from the project., i.e. CC-BY-4.0. See the note in the markdown cell above to consult full list of available licenses.\n", + "# If you have multiple licenses, you can pick 'various'\n", + "project_license = \"proprietary\"\n", + "\n", + "# Define spatial extent of the project study area in epsg:4326\n", + "# if you have multiple disjoint study areas, specify the bounding box that covers all of them\n", + "# i.e project_w, project_s, project_e, project_n = -180.0, -90.0, 180.0, 90.0\n", + "project_w = -180.0\n", + "project_s = -90.0\n", + "project_e = 180.0\n", + "project_n = 90.0\n", + "\n", + "# The project start and end times\n", + "project_start_year = 2019\n", + "project_start_month = 9\n", + "project_start_day = 24\n", + "\n", + "project_end_year = 2022\n", + "project_end_month = 10\n", + "project_end_day = 15\n", + "\n", + "# Define the links to the project websites \n", + "website_link = \"http://www.4d-antarctica.org/\" # link to project-dedicated website (hosted also on external servers)\n", + "eo4society_link = \"https://eo4society.esa.int/projects/4dantarctica/\" # link to the project website on eo4society projects list\n", + "\n", + "# Define project themes, according to OSC ontology. Pick one or more from:\n", + "# - atmosphere, cryosphere, land, magnetosphere-ionosphere, oceans, solid-earth.\n", + "# See the list here: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/blob/main/themes/catalog.json\n", + "project_themes = [\"cryosphere\"]\n", + "\n", + "# provide the Name and e-mail address to ESA Technical Officer (TO) supporting your project:\n", + "to_name = \"Diego Fernandez\"\n", + "to_email = \"diego.fernandez@esa.int\"\n", + "\n", + "# List the consortium members in a tuple with format (name, contact_email), for example - ('University A', \"contact@universitya.fr\")\n", + "consortium_members = [\n", + " (\"UNIVERSITY OF EDINBURGH (GB)\", \"\"),\n", + " (\"CNR: RESEARCH INSTITUTE FOR GEO:HYDROLOGICAL PROTECTION IRPI (IT)\", \"\"),\n", + " (\"DLR GERMAN AEROSPACE CENTER (DE)\", \"\"),\n", + " (\"EARTHWAVE Ltd (GB)\", \"\"),\n", + " (\"EIDGENOSSISCHE TECHNISCHE HOCHSCHULE ZURICH (ETH ZURICH) (CH)\", \"\"),\n", + " (\"ENVEO ENVIRONMENTAL EARTH OBSERVATION GMBH (AT)\", \"\"),\n", + " (\"SHEPHERD SPACE LTD. (GB)\", \"\"),\n", + " (\"Technical University of Denmark (DK)\", \"\"),\n", + " (\"UKRI Rutherford Appleton Laboratory (GB)\", \"\"),\n", + " (\"UNIVERSITY OF LANCASTER ENVIROMENT CENTRE (GB)\", \"\"),\n", + " (\"UNIVERSITY OF LEEDS\", \"\"),\n", + " (\"SCHOOL OF EARTH AND ENVIRONMENT (GB)\", \"\"),\n", + " (\"UNIVERSITE GRENOBLE ALPES (FR)\", \"\")\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-project-build", + "metadata": {}, + "source": [ + "Build the Project collection from the structured input. This turns the Python metadata object into the OSC/STAC collection that will be written to the catalog.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5966518b", + "metadata": {}, + "outputs": [], + "source": [ + "project_bbox = [[project_w, project_s, project_e, project_n]]\n", + "\n", + "project_metadata = ProjectCollectionMetadata(\n", + " project_id=project_id,\n", + " project_title=project_title,\n", + " project_description=project_description,\n", + " project_status=project_status,\n", + " project_license=project_license,\n", + " project_bbox=project_bbox,\n", + " project_start_datetime=datetime(project_start_year, project_start_month, project_start_day),\n", + " project_end_datetime=datetime(project_end_year, project_end_month, project_end_day),\n", + " project_themes=project_themes,\n", + " to_name=to_name,\n", + " to_email=to_email,\n", + " consortium_members=consortium_members,\n", + " website_link=website_link,\n", + " eo4society_link=eo4society_link or None,\n", + ")\n", + "\n", + "project_collection = create_project_collection(project_metadata)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "de1ec424", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# save the projection collection to the catalog fork\n", + "save_project_collection_to_osc(project_collection, catalog_root)\n" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-product-inputs", + "metadata": {}, + "source": [ + "### Create the Product metadata\n", + "\n", + "A Product describes the dataset that OSC users should be able to find and reuse. It links back to the Project and records the dataset title, description, license, keywords, spatial and temporal extent, region, themes, EO missions, variables, parameters, and DOI or access link.\n", + "\n", + "A project can have multiple products.\n", + "\n", + "Use the product extent and dates for the dataset itself, not just the wider project. If your product has several disconnected areas, provide one bounding box for each area.\n", + "\n", + "See the `guides/2.Product.ipynb` guide for more information about the project collection." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "835ad039", + "metadata": {}, + "outputs": [], + "source": [ + "# Define id, title, description, product status,license.\n", + "# A custom id of the product (must be different from project!), it can be related to the title, i.e. - 4datlantic-ohc-dataset. Use dash \"-\" symbol to separare words in the id\"\n", + "product_id = \"subglacial-lakes-boundries-2\"\n", + "product_title = \"Subglacial lakes boundaries and elevation time series\"\n", + "product_description = \"85 active Antarctic subglacial lakes detected from a decade of CryoSat-2 radar altimetry. Documenting draining and filling events and time-varying boundaries of subglacial lake activity. Ice surface elevation change (dz) for the period October 2010 to July 2020.\"\n", + "# Product status: pick from - ongoing or completed\n", + "product_status = \"completed\"\n", + "\n", + "# Define the product license. i.e. CC-BY-4.0. See the note in the markdown cell above to consult full list of available licenses. \n", + "# If you have multiple licenses, you can pick 'various'\n", + "product_license = 'CC-BY-4.0'\n", + "\n", + "# Define at most five keywords for the product. You can use any short text, that allow users to discover your product.\n", + "product_keywords = [ \n", + " \"Subglacial lakes\",\n", + " \"Glaciers\",\n", + " \"Glacial Lake Extent\",\n", + " \"Antarctica\"\n", + "] \n", + "\n", + "# Define spatial extent of PRODUCT/DATASET in epsg:4326. If the dataset covers discontinuous regions, \\\n", + "# add the bounding box boundaries for each\n", + "# i..e \n", + "# - a dataset with global coverage is:product_s product_w, product_n, product_e = [-180.0], [-90.0], [180.0], [90.0]\n", + "# - a dataset with multiple disjoint regions is: \n", + "# product_s product_w, product_n, product_e = [-180.0, -180.0], [-90.0, -90.0], [180.0, 180.0], [90.0, 90.0]\n", + "# Use bounds from your existing GeoDataFrame, ensuring EPSG:4326 for OSC product extent\n", + "\n", + "\n", + "# Keep variable names expected by the notebook's next cell (zip order is used as bbox order)\n", + "product_w = [-180.0]\n", + "product_s = [-90.0]\n", + "product_e = [180.0]\n", + "product_n = [-60.0]\n", + "\n", + "\n", + "\n", + "# Define the temporal extent of PRODUCT/ DATASET\n", + "start_dt = \"2010-07-01T00:00:00Z\"\n", + "end_dt = \"2020-07-01T00:00:00Z\"\n", + "\n", + "product_start_year, product_start_month, product_start_day = (\n", + " 2010, 7, 1\n", + ")\n", + "product_end_year, product_end_month, product_end_day = (\n", + " 2020, 7, 1\n", + ")\n", + "\n", + "\n", + "# Define the semantic region covered by this product, i.e. Belgium, Global etc. \n", + "product_region = \"Antarctica\"\n", + "\n", + "# Define product themes i.e. land. Pick one or more from:\n", + "# - atmosphere, cryosphere, land, magnetosphere-ionosphere, oceans, solid-earth.\n", + "# See the list here: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/blob/main/themes/catalog.json\n", + "product_themes = [\"cryosphere\"]\n", + "\n", + "# Define the eo-misison(s) used to generate the product. i.e. - \"sentinel-2\"\n", + "# Pick one or more from - https://github.com/ESA-EarthCODE/open-science-catalog-metadata/tree/main/eo-missions\n", + "# i.e. product_missions = ['in-situ-observations', 'grace', 'numerical-models']\n", + "product_missions = [\"cryosat-2\"]\n", + "\n", + "# Define variables describing at best your Product/ dataset: \n", + "# Pick one or more from from https://github.com/ESA-EarthCODE/open-science-catalog-metadata/tree/main/variables\n", + "# Please see the comment in the markdown cell above to select a right variable and provide it in the correct format i.e. i.e. \"crop-yield-forecast\"\n", + "product_variables = [\"subglacial-lake\"]\n", + "# Define the parameters describing your product in standardised CF convention format: i.e. \"leaf_area_index\". \n", + "# Please see the description in the markdown cell above to select a right parameter and provide it in the correct format\n", + "product_parameters = [\"subglacial-lake-extent\"]\n", + "\n", + "# Provide DOI number assigned to your product, i.e. \"https://doi.org/10.57780/s3d-83ad619\". If your product does not have one, type: 'None'\n", + "# See description in the markdown cell to request the DOI if needed. \n", + "product_doi = \"https://doi.org/10.5281/zenodo.16330564\"\n", + "\n", + "# Define the related project id and title\n", + "# These must match the new or an already existing project in the catalog! Alteratively correct links cannot be produced! \n", + "# See the previous 'project metatdata' step, and copy and paste these parameters according to your previous inputs. \n", + "project_id = project_id\n", + "project_title = project_title\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-product-build", + "metadata": {}, + "source": [ + "Build the Product collection from the input fields. The product must reference the Project created above so OSC can connect the dataset to its research context.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dd522b9f", + "metadata": {}, + "outputs": [], + "source": [ + "# create a product\n", + "product_bbox = [[w, s, e, n] for s, w, n, e in zip(product_s, product_w, product_n, product_e)]\n", + "\n", + "product_metadata = ProductCollectionMetadata(\n", + " product_id=product_id,\n", + " product_title=product_title,\n", + " product_description=product_description,\n", + " product_bbox=product_bbox,\n", + " product_start_datetime=datetime(product_start_year, product_start_month, product_start_day),\n", + " product_end_datetime=datetime(product_end_year, product_end_month, product_end_day),\n", + " product_license=product_license,\n", + " product_keywords=product_keywords,\n", + " product_status=product_status,\n", + " product_region=product_region,\n", + " product_themes=product_themes,\n", + " product_missions=product_missions,\n", + " product_variables=product_variables,\n", + " project_id=project_id,\n", + " project_title=project_title,\n", + " product_parameters=product_parameters,\n", + " product_doi=product_doi,\n", + " license_link=None,\n", + " access_link=product_doi\n", + ")\n", + "\n", + "product_collection = create_product_collection(product_metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "444095c1", + "metadata": {}, + "outputs": [], + "source": [ + "save_product_collection_to_catalog(product_collection, catalog_root)" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-workflow-inputs", + "metadata": {}, + "source": [ + "### Create the Workflow metadata\n", + "\n", + "A Workflow records the code, method, or processing logic associated with the product. This helps reviewers and future users understand how the dataset can be reproduced or inspected.\n", + "\n", + "See the `guides/3.Workflow.ipynb` guide for more information about the project collection.\n", + "\n", + "The `codeurl` should point to a stable code repository, archive, or DOI; the formats and themes should match what the workflow actually consumes and produces." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3cb48006", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "from earthcode.metadata_input_definitions import WorkflowMetadata\n", + "from earthcode.static import create_workflow_record\n", + "from earthcode.git_add import save_workflow_record_to_osc\n", + "from earthcode.validator import validate_catalog" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ef1deb06", + "metadata": {}, + "outputs": [], + "source": [ + "# BASIC INFORMATION ABOUT THE WORKFLOW\n", + "# Unique ID for the visualization/reproduction script\n", + "workflow_id = \"sgl-time-series-reproduction-2\"\n", + "workflow_title = \"Subglacial Lake Time Series Visualization Workflow\"\n", + "workflow_description = \"Workflow for reproducing ice surface elevation change figures from time series data.\" \n", + "\n", + "# Define at most five keywords for the workflow\n", + "workflow_keywords = [\n", + " \"Subglacial lakes\",\n", + " \"Glaciers\",\n", + " \"Glacial Lake Extent\",\n", + " \"Antarctica\"\n", + "] \n", + "\n", + "# Define the license of the workflow\n", + "workflow_license = 'MIT'\n", + "\n", + "# Input/Output formats: The script consumes CSV/Parquet and outputs figures (PNG/PDF)\n", + "workflow_formats = ['csv', 'geoparquet', 'png']\n", + "\n", + "\n", + "# Theme\n", + "workflow_themes = ['cryosphere']\n", + "\n", + "# Contact info (Using corresponding author from Wilson et al., 2025)\n", + "workflow_contracts_info = [('Sally F. Wilson', \"n/a\")]\n", + "\n", + "# URL to the code repository/DOI\n", + "codeurl = 'https://doi.org/10.5281/zenodo.16330564'\n", + "\n", + "\n", + "# Optional workflow record fields\n", + "workflow_doi = None\n", + "workflow_w = -180.0\n", + "workflow_s = -90.0\n", + "workflow_e = 180.0\n", + "workflow_n = 90.0\n", + "workflow_start_year = 2021\n", + "workflow_start_month = 1\n", + "workflow_start_day = 1\n", + "workflow_end_year = 2021\n", + "workflow_end_month = 12\n", + "workflow_end_day = 31\n", + "include_workflow_bbox = False\n", + "include_workflow_time = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "osc-tutorial-workflow-save", + "metadata": {}, + "outputs": [], + "source": [ + "workflow_metadata = WorkflowMetadata(\n", + " workflow_id=workflow_id,\n", + " workflow_title=workflow_title,\n", + " workflow_description=workflow_description,\n", + " workflow_license=workflow_license,\n", + " workflow_keywords=workflow_keywords,\n", + " workflow_formats=workflow_formats,\n", + " workflow_themes=workflow_themes,\n", + " codeurl=codeurl,\n", + " project_id=project_id,\n", + " project_title=project_title,\n", + " workflow_doi=workflow_doi,\n", + " workflow_bbox=[[workflow_w, workflow_s, workflow_e, workflow_n]] if include_workflow_bbox else None,\n", + " workflow_start_datetime=datetime(workflow_start_year, workflow_start_month, workflow_start_day) if include_workflow_time else None,\n", + " workflow_end_datetime=datetime(workflow_end_year, workflow_end_month, workflow_end_day) if include_workflow_time else None,\n", + ")\n", + "\n", + "workflow_record = create_workflow_record(workflow_metadata)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "fd33d971", + "metadata": {}, + "outputs": [], + "source": [ + "save_workflow_record_to_osc(workflow_record, catalog_root)\n" + ] + }, + { + "cell_type": "markdown", + "id": "939fe0a0", + "metadata": {}, + "source": [ + "### Add an experiment\n", + "\n", + "An experiment links a product and a workflow together, showing that a particular dataset is the output of a specific run of an algorithm.\n", + "\n", + "More information is available in `docs/guides/5.Experiment.ipynb`." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "31d6ac69", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "from earthcode.metadata_input_definitions import ExperimentMetadata\n", + "from earthcode.static import create_experiment_record\n", + "from earthcode.git_add import save_experiment_record_to_osc" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0ce03a6e", + "metadata": {}, + "outputs": [], + "source": [ + "# experiment info\n", + "# Experiment id\n", + "experiment_id = workflow_id = \"sgl-time-series-reproduction-experiment\"\n", + "experiment_title = \"Subglacial Lake Time Series Visualization Experiment\"\n", + "experiment_description = \"Experiment for the subglacial dataset.\" \n", + "\n", + "experiment_license = \"CC-BY-4.0\"\n", + "experiment_keywords = [\n", + " \"Subglacial lakes\",\n", + " \"Glaciers\",\n", + " \"Glacial Lake Extent\",\n", + " \"Antarctica\"\n", + "]\n", + "\n", + "# Define the input/output formats that this experiment works with\n", + "# i.e. GeoTIFF, Zarr, netCDF, etc\n", + "experiment_formats = ['safefile', \"parquet\"]\n", + "\n", + "# Define themes i.e. land. Pick one or more from:\n", + "# - atmosphere, cryosphere, land, magnetosphere-ionosphere, oceans, solid-earth.\n", + "experiment_themes = [\"cryosphere\"]\n", + "\n", + "# link to the specification of the input parameters for the experiment\n", + "experiment_input_parameters_link = codeurl\n", + "# link to the enviroment in which the experiment was performed\n", + "experiment_enviroment_link = codeurl\n", + "\n", + "## ID and title of the associated workflow\n", + "workflow_id = workflow_id\n", + "workflow_title = workflow_title\n", + "\n", + "## ID and title title of the associated product\n", + "product_id = product_id\n", + "product_title = product_title\n", + "\n", + "# Optional experiment record fields\n", + "experiment_contacts = None\n", + "\n", + "experiment_w = -180.0\n", + "experiment_s = -90.0\n", + "experiment_e = 180.0\n", + "experiment_n = 90.0\n", + "experiment_start_year = 2021\n", + "experiment_start_month = 1\n", + "experiment_start_day = 1\n", + "experiment_end_year = 2021\n", + "experiment_end_month = 12\n", + "experiment_end_day = 31\n", + "\n", + "include_experiment_bbox = False\n", + "include_experiment_time = False" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "dbd76217", + "metadata": {}, + "outputs": [], + "source": [ + "catalog_root = Path(catalog_root)\n", + "\n", + "experiment_metadata = ExperimentMetadata(\n", + " experiment_id=experiment_id,\n", + " experiment_title=experiment_title,\n", + " experiment_description=experiment_description,\n", + " experiment_license=experiment_license,\n", + " experiment_keywords=experiment_keywords,\n", + " experiment_formats=experiment_formats,\n", + " experiment_themes=experiment_themes,\n", + " experiment_input_parameters_link=experiment_input_parameters_link,\n", + " experiment_enviroment_link=experiment_enviroment_link,\n", + " workflow_id=workflow_id,\n", + " workflow_title=workflow_title,\n", + " product_id=product_id,\n", + " product_title=product_title,\n", + " contacts=experiment_contacts,\n", + " experiment_bbox=[[experiment_w, experiment_s, experiment_e, experiment_n]] if include_experiment_bbox else None,\n", + " experiment_start_datetime=datetime(experiment_start_year, experiment_start_month, experiment_start_day) if include_experiment_time else None,\n", + " experiment_end_datetime=datetime(experiment_end_year, experiment_end_month, experiment_end_day) if include_experiment_time else None,\n", + ")\n", + "\n", + "experiment_record = create_experiment_record(experiment_metadata)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "f6fa7e32", + "metadata": {}, + "outputs": [], + "source": [ + "save_experiment_record_to_osc(experiment_record, catalog_root)" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-file-overview", + "metadata": {}, + "source": [ + "## Step 3. Add file-level metadata as STAC Items\n", + "\n", + "Product metadata describes the dataset as a whole. STAC Item metadata describes the individual files or assets that make up that dataset.\n", + "\n", + "In this section you will create one STAC Item for a small Parquet asset, add projection and column metadata, attach the asset URL, validate the item, convert it to the EarthCODE-flavoured item, and save it under the product collection. \n", + "\n", + "If your own contribution has multiple files, you will have to create a STAC item for each file and add it to a collection.\n", + "\n", + "See the guides for specific examples of how to generate metadata for the three most common data types:\n", + "- Parquet - `docs/guides/parquet_file_metadata.ipynb`\n", + "- Zarr - `docs/guides/zarr_file_metadata.ipynb`\n", + "- COG - `docs/guides/cog_file_metadata.ipynb`" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "60822aa7", + "metadata": {}, + "outputs": [], + "source": [ + "import geopandas as gpd\n", + "import pandas as pd\n", + "from shapely.geometry import box, mapping\n", + "import pystac\n", + "from pystac.extensions.projection import ProjectionExtension\n", + "import datetime\n", + "import fsspec\n", + "import io" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "829afa5a", + "metadata": {}, + "outputs": [], + "source": [ + "data_url = \"https://s3.waw4-1.cloudferro.com/EarthCODE/OSCAssets/polar_cube_datasets/supraglacial_lakes/WAIS_Jan_2017_Polygons.parquet\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "b12c0aeb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
POLY_AREACENTROID_XCENTROID_YAreaShape_LengShape_AreaBedrockCoastGL_ON_ISLocation...PolsbyPoppFractalReockSchwartzbeLW_RatioFeature_ClElevationSlopeSpeedgeometry
02250.000000-1.122141e+06-1.179198e+062298.467416240.000002250.000000439.23672016041.3806340.000000Grounded Ice...0.4908741.4083550.5063000.7006240.857143Lake3601.88421061.69920POLYGON ((-1122127.5 -1179172.5, -1122127.5 -1...
125200.000000-1.122477e+06-1.178988e+0625742.7262682160.0000025200.000000415.75923816085.3454990.000000Grounded Ice...0.0678741.3199760.0527960.2605260.114516Channel3601.88421060.73470MULTIPOLYGON (((-1122607.5 -1178917.5, -112265...
2675.000000-1.122930e+06-1.178760e+06689.534115120.00000675.000000740.34162416507.5119530.000000Grounded Ice...0.5890491.3607780.3819720.7674950.333333Lake3882.73161039.97260POLYGON ((-1122907.5 -1178767.5, -1122952.5 -1...
311250.000000-1.123134e+06-1.178679e+0611492.233178930.0000011250.000000770.66189316537.8535580.000000Grounded Ice...0.1634541.3647220.1126760.4042950.186667Lake3882.73161046.55940POLYGON ((-1123267.5 -1178602.5, -1123267.5 -1...
4198.750411-1.809312e+067.091160e+05200.15923859.81225198.75041114427.999008134873.872142418.393202Ice Shelf...0.6981321.2935170.5092940.8355430.500000Lake370.1219649.98336POLYGON ((-1809304.184 709123.549, -1809311.33...
..................................................................
104732025.0000002.573867e+05-7.738833e+052121.704652240.000002025.0000000.000000496657.6870600.000000Grounded Ice...0.4417861.3891310.4491990.6646700.666667Lake5027.21503047.97920POLYGON ((257407.5 -773857.5, 257407.5 -773872...
10474450.0000002.574525e+05-7.738425e+05471.491754120.00000450.0000000.000000496703.6802790.000000Grounded Ice...0.3926991.2760850.3183100.6266570.500000Lake5027.21503047.97920MULTIPOLYGON (((257452.5 -773842.5, 257452.5 -...
104753375.0000002.576260e+05-7.736940e+053536.188110480.000003375.00000074.210787496768.4042990.000000Grounded Ice...0.1840781.3159110.1690140.4290430.333333Lake4717.99781056.07010MULTIPOLYGON (((257587.5 -773752.5, 257587.5 -...
104766750.0000002.577890e+05-7.736955e+057072.372180630.000006750.000000135.624326496743.4612250.000000Grounded Ice...0.2137141.3679310.2329100.4622920.444444Lake4717.99781056.07010POLYGON ((257827.5 -773677.5, 257827.5 -773707...
10477900.0000002.577712e+05-7.736138e+05942.989698150.00000900.00000089.562774496867.0058180.000000Grounded Ice...0.5026551.3575910.3917660.7089820.666667Lake4717.99781036.09830POLYGON ((257767.5 -773632.5, 257752.5 -773632...
\n", + "

10478 rows × 25 columns

\n", + "
" + ], + "text/plain": [ + " POLY_AREA CENTROID_X CENTROID_Y Area Shape_Leng \\\n", + "0 2250.000000 -1.122141e+06 -1.179198e+06 2298.467416 240.00000 \n", + "1 25200.000000 -1.122477e+06 -1.178988e+06 25742.726268 2160.00000 \n", + "2 675.000000 -1.122930e+06 -1.178760e+06 689.534115 120.00000 \n", + "3 11250.000000 -1.123134e+06 -1.178679e+06 11492.233178 930.00000 \n", + "4 198.750411 -1.809312e+06 7.091160e+05 200.159238 59.81225 \n", + "... ... ... ... ... ... \n", + "10473 2025.000000 2.573867e+05 -7.738833e+05 2121.704652 240.00000 \n", + "10474 450.000000 2.574525e+05 -7.738425e+05 471.491754 120.00000 \n", + "10475 3375.000000 2.576260e+05 -7.736940e+05 3536.188110 480.00000 \n", + "10476 6750.000000 2.577890e+05 -7.736955e+05 7072.372180 630.00000 \n", + "10477 900.000000 2.577712e+05 -7.736138e+05 942.989698 150.00000 \n", + "\n", + " Shape_Area Bedrock Coast GL_ON_IS Location \\\n", + "0 2250.000000 439.236720 16041.380634 0.000000 Grounded Ice \n", + "1 25200.000000 415.759238 16085.345499 0.000000 Grounded Ice \n", + "2 675.000000 740.341624 16507.511953 0.000000 Grounded Ice \n", + "3 11250.000000 770.661893 16537.853558 0.000000 Grounded Ice \n", + "4 198.750411 14427.999008 134873.872142 418.393202 Ice Shelf \n", + "... ... ... ... ... ... \n", + "10473 2025.000000 0.000000 496657.687060 0.000000 Grounded Ice \n", + "10474 450.000000 0.000000 496703.680279 0.000000 Grounded Ice \n", + "10475 3375.000000 74.210787 496768.404299 0.000000 Grounded Ice \n", + "10476 6750.000000 135.624326 496743.461225 0.000000 Grounded Ice \n", + "10477 900.000000 89.562774 496867.005818 0.000000 Grounded Ice \n", + "\n", + " ... PolsbyPopp Fractal Reock Schwartzbe LW_Ratio Feature_Cl \\\n", + "0 ... 0.490874 1.408355 0.506300 0.700624 0.857143 Lake \n", + "1 ... 0.067874 1.319976 0.052796 0.260526 0.114516 Channel \n", + "2 ... 0.589049 1.360778 0.381972 0.767495 0.333333 Lake \n", + "3 ... 0.163454 1.364722 0.112676 0.404295 0.186667 Lake \n", + "4 ... 0.698132 1.293517 0.509294 0.835543 0.500000 Lake \n", + "... ... ... ... ... ... ... ... \n", + "10473 ... 0.441786 1.389131 0.449199 0.664670 0.666667 Lake \n", + "10474 ... 0.392699 1.276085 0.318310 0.626657 0.500000 Lake \n", + "10475 ... 0.184078 1.315911 0.169014 0.429043 0.333333 Lake \n", + "10476 ... 0.213714 1.367931 0.232910 0.462292 0.444444 Lake \n", + "10477 ... 0.502655 1.357591 0.391766 0.708982 0.666667 Lake \n", + "\n", + " Elevation Slope Speed \\\n", + "0 360 1.884210 61.69920 \n", + "1 360 1.884210 60.73470 \n", + "2 388 2.731610 39.97260 \n", + "3 388 2.731610 46.55940 \n", + "4 37 0.121964 9.98336 \n", + "... ... ... ... \n", + "10473 502 7.215030 47.97920 \n", + "10474 502 7.215030 47.97920 \n", + "10475 471 7.997810 56.07010 \n", + "10476 471 7.997810 56.07010 \n", + "10477 471 7.997810 36.09830 \n", + "\n", + " geometry \n", + "0 POLYGON ((-1122127.5 -1179172.5, -1122127.5 -1... \n", + "1 MULTIPOLYGON (((-1122607.5 -1178917.5, -112265... \n", + "2 POLYGON ((-1122907.5 -1178767.5, -1122952.5 -1... \n", + "3 POLYGON ((-1123267.5 -1178602.5, -1123267.5 -1... \n", + "4 POLYGON ((-1809304.184 709123.549, -1809311.33... \n", + "... ... \n", + "10473 POLYGON ((257407.5 -773857.5, 257407.5 -773872... \n", + "10474 MULTIPOLYGON (((257452.5 -773842.5, 257452.5 -... \n", + "10475 MULTIPOLYGON (((257587.5 -773752.5, 257587.5 -... \n", + "10476 POLYGON ((257827.5 -773677.5, 257827.5 -773707... \n", + "10477 POLYGON ((257767.5 -773632.5, 257752.5 -773632... \n", + "\n", + "[10478 rows x 25 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# this is a small file, so we read it in one go\n", + "with fsspec.open(data_url, mode=\"rb\") as f:\n", + " # Read the entire streaming file into memory\n", + " file_bytes = f.read()\n", + " \n", + " # Wrap it in a BytesIO object, which PyArrow can seek through\n", + " buffer = io.BytesIO(file_bytes)\n", + " \n", + " # Read the parquet from the memory buffer\n", + " gdf = gpd.read_parquet(buffer)\n", + "\n", + "gdf" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "da93828b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=supraglacial-lakes-boundries-only>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# read the dataset spatial and temporal boundary\n", + "minx, miny, maxx, maxy = gdf.total_bounds\n", + "start, end = (datetime.datetime(2017, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),\n", + " datetime.datetime(2017, 12, 31, 0, 0, tzinfo=datetime.timezone.utc))\n", + "\n", + "item = pystac.Item(\n", + " id=\"supraglacial-lakes-boundries-only\",\n", + " geometry=mapping(box(minx, miny, maxx, maxy)),\n", + " bbox=[minx, miny, maxx, maxy],\n", + " datetime=start,\n", + " properties={'title': 'Supraglacial lakes boundaries'},\n", + " \n", + ")\n", + "\n", + "proj = ProjectionExtension.ext(item, add_if_missing=True)\n", + "proj.code = gdf.crs.to_string() # e.g. \"EPSG:3031\"\n", + "proj.wkt2 = gdf.crs.to_wkt()\n", + "proj.bbox = [minx, miny, maxx, maxy] # native CRS bbox\n", + "proj.geometry = mapping(box(minx, miny, maxx, maxy))\n", + "\n", + "item.extra_fields['title'] = 'Subglacial lakes boundaries data'\n", + "item" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "9267aece", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=supraglacial-lakes-boundries-only>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "asset = pystac.Asset(href=data_url, media_type=\"application/x-parquet\", roles=[\"data\"])\n", + "\n", + "asset.extra_fields[\"parquet:columns\"] = [\n", + " {\n", + " \"name\": \"POLY_AREA\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Polygon area\",\n", + " \"description\": (\n", + " \"Area of the supraglacial lake polygon as provided in the source vector dataset. \"\n", + " \"This may duplicate or closely correspond to Shape_Area or Area, depending on the \"\n", + " \"source processing workflow.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"CENTROID_X\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake centroid longitude\",\n", + " \"description\": (\n", + " \"X coordinate of the supraglacial lake polygon centroid. For Antarctic geographic \"\n", + " \"datasets this is typically longitude in decimal degrees, unless the source layer \"\n", + " \"uses a projected coordinate reference system.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"CENTROID_Y\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake centroid latitude\",\n", + " \"description\": (\n", + " \"Y coordinate of the supraglacial lake polygon centroid. For Antarctic geographic \"\n", + " \"datasets this is typically latitude in decimal degrees, unless the source layer \"\n", + " \"uses a projected coordinate reference system.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"Area\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Supraglacial lake area\",\n", + " \"description\": (\n", + " \"Surface area of the mapped supraglacial lake feature. Used to describe lake size \"\n", + " \"and to support derived metrics such as volume estimates and area-perimeter ratios.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"Shape_Leng\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake polygon perimeter\",\n", + " \"description\": (\n", + " \"Perimeter or boundary length of the supraglacial lake polygon geometry.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Shape_Area\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake polygon shape area\",\n", + " \"description\": (\n", + " \"Area of the lake polygon geometry calculated by the source GIS software. This may \"\n", + " \"duplicate or closely correspond to POLY_AREA or Area.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"Bedrock\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to exposed bedrock\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the nearest mapped exposed bedrock \"\n", + " \"or rock outcrop, where available.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Coast\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to coast\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the Antarctic coastline or ice-shelf \"\n", + " \"front used in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"GL_ON_IS\",\n", + " \"type\": \"boolean\",\n", + " \"long_name\": \"Grounding-line-on-ice-shelf flag\",\n", + " \"description\": (\n", + " \"Flag indicating whether the supraglacial lake is located on an ice shelf or in a \"\n", + " \"grounding-line-related ice-shelf setting, according to the source classification.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"Location\",\n", + " \"type\": \"string\",\n", + " \"long_name\": \"Lake location class\",\n", + " \"description\": (\n", + " \"Textual location category or regional setting assigned to the supraglacial lake, \"\n", + " \"such as ice sheet, ice shelf, grounding-zone, coastal, or named regional grouping, \"\n", + " \"depending on the source classification.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"GL\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to grounding line\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the Antarctic grounding line used \"\n", + " \"in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Volume\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Estimated lake volume\",\n", + " \"description\": (\n", + " \"Estimated water volume of the supraglacial lake feature, typically derived from \"\n", + " \"mapped lake area and an empirical or remotely sensed depth relationship.\"\n", + " ),\n", + " \"unit\": \"m3\",\n", + " },\n", + " {\n", + " \"name\": \"A_P_Ratio\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Area-perimeter ratio\",\n", + " \"description\": (\n", + " \"Ratio of lake polygon area to perimeter, used as a compactness or shape metric for \"\n", + " \"the mapped supraglacial lake.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"REMA_Max\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Maximum REMA elevation\",\n", + " \"description\": (\n", + " \"Maximum surface elevation from the Reference Elevation Model of Antarctica within \"\n", + " \"or associated with the supraglacial lake polygon.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"REMA_Min\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Minimum REMA elevation\",\n", + " \"description\": (\n", + " \"Minimum surface elevation from the Reference Elevation Model of Antarctica within \"\n", + " \"or associated with the supraglacial lake polygon.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"PolsbyPopp\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Polsby-Popper compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric for the lake polygon based on the ratio of polygon area \"\n", + " \"to the area of a circle with the same perimeter. Values closer to 1 indicate a more \"\n", + " \"compact, circular shape.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Fractal\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Fractal dimension shape index\",\n", + " \"description\": (\n", + " \"Shape-complexity metric describing the irregularity of the supraglacial lake polygon \"\n", + " \"boundary. Higher values generally indicate a more complex or convoluted outline.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Reock\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Reock compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric comparing the polygon area with the area of its minimum \"\n", + " \"bounding circle. Values closer to 1 indicate a more compact shape.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Schwartzbe\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Schwartzberg compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric comparing the polygon perimeter with the circumference of \"\n", + " \"a circle of equal area. Values closer to 1 generally indicate a more compact, \"\n", + " \"circular polygon.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"LW_Ratio\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Length-width ratio\",\n", + " \"description\": (\n", + " \"Ratio of lake length to lake width, used to describe elongation of the supraglacial \"\n", + " \"lake polygon.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Feature_Cl\",\n", + " \"type\": \"string\",\n", + " \"long_name\": \"Feature class\",\n", + " \"description\": (\n", + " \"Classification assigned to the mapped feature in the source dataset. For this asset, \"\n", + " \"features represent supraglacial lakes, but this field may preserve source-level \"\n", + " \"feature class labels or quality classes.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"Elevation\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake surface elevation\",\n", + " \"description\": (\n", + " \"Representative surface elevation associated with the supraglacial lake feature, \"\n", + " \"likely derived from REMA or another Antarctic digital elevation model.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Slope\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Surface slope\",\n", + " \"description\": (\n", + " \"Representative ice-surface slope associated with the supraglacial lake feature.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"Speed\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Ice surface velocity\",\n", + " \"description\": (\n", + " \"Representative ice-flow speed at or near the supraglacial lake feature, derived from \"\n", + " \"an Antarctic ice-velocity product used in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m yr-1\",\n", + " },\n", + " {\n", + " \"name\": \"geometry\",\n", + " \"type\": \"geometry\",\n", + " \"long_name\": \"Supraglacial lake polygon geometry\",\n", + " \"description\": (\n", + " \"Polygon geometry representing the mapped supraglacial lake surface extent. Stored as \"\n", + " \"the GeoParquet geometry column.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + "]\n", + "\n", + "item.add_asset('data', asset)\n", + "item.validate()\n", + "item" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-earthcode-convert", + "metadata": {}, + "source": [ + "### Save the item under the Product collection\n", + "\n", + "The generic STAC Item now needs the EarthCODE catalog links that connect it to the product. Convert it, then save it into the product collection in your local catalog checkout.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "b01f29aa", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from pathlib import Path\n", + "from earthcode.static import generic_stac_item_to_earthcode_stac_item\n", + "from earthcode.git_add import save_item_to_product_collection" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "9d218b4d", + "metadata": {}, + "outputs": [], + "source": [ + "# convert to earthcode item with a reference to a specific open science catalog collection\n", + "earthcode_item = generic_stac_item_to_earthcode_stac_item(item, product_id, asset_key='data')" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "1bcedcb5", + "metadata": {}, + "outputs": [], + "source": [ + "# save the item to the collection\n", + "save_item_to_product_collection(earthcode_item, product_id, catalog_root)" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-validation", + "metadata": {}, + "source": [ + "## Step 4. Validate the local catalog changes\n", + "\n", + "Run validation after all project, product, workflow, experiment, and file-level metadata for your real contribution have been saved to the open-science-catalog-metadata directory.\n", + "\n", + "The tutorial raises an error if validation finds problems, so you do not continue with a broken local catalog.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "6e7c0a79", + "metadata": {}, + "outputs": [], + "source": [ + "from earthcode.validator import validate_catalog\n", + "\n", + "errors, error_files = validate_catalog(catalog_root)\n", + "if errors or error_files:\n", + " raise AssertionError(f\"Catalog validation failed. errors={len(errors)} files={len(error_files)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-validation-output", + "metadata": {}, + "source": [ + "If validation fails, inspect the returned `errors` and `error_files` to find the records that need to be fixed.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "ea453bd0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "errors" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "e8357e59", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "error_files" + ] + }, + { + "cell_type": "markdown", + "id": "osc-tutorial-pr", + "metadata": {}, + "source": [ + "## Step 5. Use the pattern for a real pull request\n", + "\n", + "To finish your contribution, you have to:\n", + "- Commit the changed files to your local `open-science-catalog-metadata` clone.\n", + "- Open a pull request against the main `open-science-catalog-metadata` repository\n", + "- Then your PR will be reviewed and after any requested changes are made, merged into the Open Science Catalog repository.\n", + "\n", + "\n", + " > ** Do not open a pull request with this tutorial output. **\n", + "\n", + "\n", + "More details on openning the PR is available in the `guides/0.Prerequisites-local.ipynb` guide\n", + "\n", + "\n", + "## Next steps:\n", + "For a real OSC contribution, repeat the same sequence with your own metadata and stable data URLs:\n", + "\n", + "1. Create Project metadata.\n", + "2. Create the Product metadata for each dataset and link it all Products to your Project.\n", + "3. Add Workflow metadata for reusable code or processing logic.\n", + "4. Add Experiment metadata when a specific workflow run generated the product.\n", + "5. Add file-level STAC Items for the all of your data files.\n", + "6. Run validation and fix any reported errors.\n", + "7. Open a pull request against `open-science-catalog-metadata` with only the intended metadata changes.\n", + "\n", + "The pull request should explain what is being added, where the data is hosted, which licenses apply, and any review notes the EarthCODE Data Steward team needs.\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/parquet_file_metadata.ipynb b/docs/tutorials/parquet_file_metadata.ipynb new file mode 100644 index 0000000..6829da6 --- /dev/null +++ b/docs/tutorials/parquet_file_metadata.ipynb @@ -0,0 +1,3978 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5ca54e6a", + "metadata": {}, + "source": [ + "# GeoParquet STAC Metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "6a8ef164", + "metadata": {}, + "outputs": [], + "source": [ + "import geopandas as gpd\n", + "import pandas as pd\n", + "from shapely.geometry import box, mapping\n", + "import pystac\n", + "from pystac.extensions.projection import ProjectionExtension\n", + "import datetime\n", + "import fsspec\n", + "import io" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3d16522c", + "metadata": {}, + "outputs": [], + "source": [ + "data_url = \"https://s3.waw4-1.cloudferro.com/EarthCODE/OSCAssets/polar_cube_datasets/supraglacial_lakes/WAIS_Jan_2017_Polygons.parquet\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0bee8357", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
POLY_AREACENTROID_XCENTROID_YAreaShape_LengShape_AreaBedrockCoastGL_ON_ISLocation...PolsbyPoppFractalReockSchwartzbeLW_RatioFeature_ClElevationSlopeSpeedgeometry
02250.000000-1.122141e+06-1.179198e+062298.467416240.000002250.000000439.23672016041.3806340.000000Grounded Ice...0.4908741.4083550.5063000.7006240.857143Lake3601.88421061.69920POLYGON ((-1122127.5 -1179172.5, -1122127.5 -1...
125200.000000-1.122477e+06-1.178988e+0625742.7262682160.0000025200.000000415.75923816085.3454990.000000Grounded Ice...0.0678741.3199760.0527960.2605260.114516Channel3601.88421060.73470MULTIPOLYGON (((-1122607.5 -1178917.5, -112265...
2675.000000-1.122930e+06-1.178760e+06689.534115120.00000675.000000740.34162416507.5119530.000000Grounded Ice...0.5890491.3607780.3819720.7674950.333333Lake3882.73161039.97260POLYGON ((-1122907.5 -1178767.5, -1122952.5 -1...
311250.000000-1.123134e+06-1.178679e+0611492.233178930.0000011250.000000770.66189316537.8535580.000000Grounded Ice...0.1634541.3647220.1126760.4042950.186667Lake3882.73161046.55940POLYGON ((-1123267.5 -1178602.5, -1123267.5 -1...
4198.750411-1.809312e+067.091160e+05200.15923859.81225198.75041114427.999008134873.872142418.393202Ice Shelf...0.6981321.2935170.5092940.8355430.500000Lake370.1219649.98336POLYGON ((-1809304.184 709123.549, -1809311.33...
..................................................................
104732025.0000002.573867e+05-7.738833e+052121.704652240.000002025.0000000.000000496657.6870600.000000Grounded Ice...0.4417861.3891310.4491990.6646700.666667Lake5027.21503047.97920POLYGON ((257407.5 -773857.5, 257407.5 -773872...
10474450.0000002.574525e+05-7.738425e+05471.491754120.00000450.0000000.000000496703.6802790.000000Grounded Ice...0.3926991.2760850.3183100.6266570.500000Lake5027.21503047.97920MULTIPOLYGON (((257452.5 -773842.5, 257452.5 -...
104753375.0000002.576260e+05-7.736940e+053536.188110480.000003375.00000074.210787496768.4042990.000000Grounded Ice...0.1840781.3159110.1690140.4290430.333333Lake4717.99781056.07010MULTIPOLYGON (((257587.5 -773752.5, 257587.5 -...
104766750.0000002.577890e+05-7.736955e+057072.372180630.000006750.000000135.624326496743.4612250.000000Grounded Ice...0.2137141.3679310.2329100.4622920.444444Lake4717.99781056.07010POLYGON ((257827.5 -773677.5, 257827.5 -773707...
10477900.0000002.577712e+05-7.736138e+05942.989698150.00000900.00000089.562774496867.0058180.000000Grounded Ice...0.5026551.3575910.3917660.7089820.666667Lake4717.99781036.09830POLYGON ((257767.5 -773632.5, 257752.5 -773632...
\n", + "

10478 rows × 25 columns

\n", + "
" + ], + "text/plain": [ + " POLY_AREA CENTROID_X CENTROID_Y Area Shape_Leng \\\n", + "0 2250.000000 -1.122141e+06 -1.179198e+06 2298.467416 240.00000 \n", + "1 25200.000000 -1.122477e+06 -1.178988e+06 25742.726268 2160.00000 \n", + "2 675.000000 -1.122930e+06 -1.178760e+06 689.534115 120.00000 \n", + "3 11250.000000 -1.123134e+06 -1.178679e+06 11492.233178 930.00000 \n", + "4 198.750411 -1.809312e+06 7.091160e+05 200.159238 59.81225 \n", + "... ... ... ... ... ... \n", + "10473 2025.000000 2.573867e+05 -7.738833e+05 2121.704652 240.00000 \n", + "10474 450.000000 2.574525e+05 -7.738425e+05 471.491754 120.00000 \n", + "10475 3375.000000 2.576260e+05 -7.736940e+05 3536.188110 480.00000 \n", + "10476 6750.000000 2.577890e+05 -7.736955e+05 7072.372180 630.00000 \n", + "10477 900.000000 2.577712e+05 -7.736138e+05 942.989698 150.00000 \n", + "\n", + " Shape_Area Bedrock Coast GL_ON_IS Location \\\n", + "0 2250.000000 439.236720 16041.380634 0.000000 Grounded Ice \n", + "1 25200.000000 415.759238 16085.345499 0.000000 Grounded Ice \n", + "2 675.000000 740.341624 16507.511953 0.000000 Grounded Ice \n", + "3 11250.000000 770.661893 16537.853558 0.000000 Grounded Ice \n", + "4 198.750411 14427.999008 134873.872142 418.393202 Ice Shelf \n", + "... ... ... ... ... ... \n", + "10473 2025.000000 0.000000 496657.687060 0.000000 Grounded Ice \n", + "10474 450.000000 0.000000 496703.680279 0.000000 Grounded Ice \n", + "10475 3375.000000 74.210787 496768.404299 0.000000 Grounded Ice \n", + "10476 6750.000000 135.624326 496743.461225 0.000000 Grounded Ice \n", + "10477 900.000000 89.562774 496867.005818 0.000000 Grounded Ice \n", + "\n", + " ... PolsbyPopp Fractal Reock Schwartzbe LW_Ratio Feature_Cl \\\n", + "0 ... 0.490874 1.408355 0.506300 0.700624 0.857143 Lake \n", + "1 ... 0.067874 1.319976 0.052796 0.260526 0.114516 Channel \n", + "2 ... 0.589049 1.360778 0.381972 0.767495 0.333333 Lake \n", + "3 ... 0.163454 1.364722 0.112676 0.404295 0.186667 Lake \n", + "4 ... 0.698132 1.293517 0.509294 0.835543 0.500000 Lake \n", + "... ... ... ... ... ... ... ... \n", + "10473 ... 0.441786 1.389131 0.449199 0.664670 0.666667 Lake \n", + "10474 ... 0.392699 1.276085 0.318310 0.626657 0.500000 Lake \n", + "10475 ... 0.184078 1.315911 0.169014 0.429043 0.333333 Lake \n", + "10476 ... 0.213714 1.367931 0.232910 0.462292 0.444444 Lake \n", + "10477 ... 0.502655 1.357591 0.391766 0.708982 0.666667 Lake \n", + "\n", + " Elevation Slope Speed \\\n", + "0 360 1.884210 61.69920 \n", + "1 360 1.884210 60.73470 \n", + "2 388 2.731610 39.97260 \n", + "3 388 2.731610 46.55940 \n", + "4 37 0.121964 9.98336 \n", + "... ... ... ... \n", + "10473 502 7.215030 47.97920 \n", + "10474 502 7.215030 47.97920 \n", + "10475 471 7.997810 56.07010 \n", + "10476 471 7.997810 56.07010 \n", + "10477 471 7.997810 36.09830 \n", + "\n", + " geometry \n", + "0 POLYGON ((-1122127.5 -1179172.5, -1122127.5 -1... \n", + "1 MULTIPOLYGON (((-1122607.5 -1178917.5, -112265... \n", + "2 POLYGON ((-1122907.5 -1178767.5, -1122952.5 -1... \n", + "3 POLYGON ((-1123267.5 -1178602.5, -1123267.5 -1... \n", + "4 POLYGON ((-1809304.184 709123.549, -1809311.33... \n", + "... ... \n", + "10473 POLYGON ((257407.5 -773857.5, 257407.5 -773872... \n", + "10474 MULTIPOLYGON (((257452.5 -773842.5, 257452.5 -... \n", + "10475 MULTIPOLYGON (((257587.5 -773752.5, 257587.5 -... \n", + "10476 POLYGON ((257827.5 -773677.5, 257827.5 -773707... \n", + "10477 POLYGON ((257767.5 -773632.5, 257752.5 -773632... \n", + "\n", + "[10478 rows x 25 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# this is a small file, so we read it in one go\n", + "with fsspec.open(data_url, mode=\"rb\") as f:\n", + " # Read the entire streaming file into memory\n", + " file_bytes = f.read()\n", + " \n", + " # Wrap it in a BytesIO object, which PyArrow can seek through\n", + " buffer = io.BytesIO(file_bytes)\n", + " \n", + " # Read the parquet from the memory buffer\n", + " gdf = gpd.read_parquet(buffer)\n", + "\n", + "gdf" + ] + }, + { + "cell_type": "markdown", + "id": "a525508a", + "metadata": {}, + "source": [ + "## Create The STAC Item And Projection Metadata\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6bf9bdb7", + "metadata": {}, + "outputs": [], + "source": [ + "minx, miny, maxx, maxy = gdf.total_bounds\n", + "start, end = (datetime.datetime(2017, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),\n", + " datetime.datetime(2017, 12, 31, 0, 0, tzinfo=datetime.timezone.utc))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "777b5a49", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=supraglacial-lakes-boundries-only>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item = pystac.Item(\n", + " id=\"supraglacial-lakes-boundries-only\",\n", + " geometry=mapping(box(minx, miny, maxx, maxy)),\n", + " bbox=[minx, miny, maxx, maxy],\n", + " datetime=start,\n", + " properties={'title': 'Supraglacial lakes boundaries'},\n", + " \n", + ")\n", + "\n", + "proj = ProjectionExtension.ext(item, add_if_missing=True)\n", + "proj.code = gdf.crs.to_string() # e.g. \"EPSG:3031\"\n", + "proj.wkt2 = gdf.crs.to_wkt()\n", + "proj.bbox = [minx, miny, maxx, maxy] # native CRS bbox\n", + "proj.geometry = mapping(box(minx, miny, maxx, maxy))\n", + "\n", + "item.extra_fields['title'] = 'Subglacial lakes boundaries data'\n", + "item" + ] + }, + { + "cell_type": "markdown", + "id": "aed0ad65", + "metadata": {}, + "source": [ + "## Create The Parquet Asset\n", + "\n", + "The asset records where the Parquet file will live and identifies it as a data asset. The href remains the original placeholder until the object-store path is supplied.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "39abd7eb", + "metadata": {}, + "outputs": [], + "source": [ + "asset = pystac.Asset(href=data_url, media_type=\"application/x-parquet\", roles=[\"data\"])" + ] + }, + { + "cell_type": "markdown", + "id": "f7822217", + "metadata": {}, + "source": [ + "## Describe The Parquet Columns\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e25fbc6c", + "metadata": {}, + "outputs": [], + "source": [ + "asset.extra_fields[\"parquet:columns\"] = [\n", + " {\n", + " \"name\": \"POLY_AREA\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Polygon area\",\n", + " \"description\": (\n", + " \"Area of the supraglacial lake polygon as provided in the source vector dataset. \"\n", + " \"This may duplicate or closely correspond to Shape_Area or Area, depending on the \"\n", + " \"source processing workflow.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"CENTROID_X\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake centroid longitude\",\n", + " \"description\": (\n", + " \"X coordinate of the supraglacial lake polygon centroid. For Antarctic geographic \"\n", + " \"datasets this is typically longitude in decimal degrees, unless the source layer \"\n", + " \"uses a projected coordinate reference system.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"CENTROID_Y\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake centroid latitude\",\n", + " \"description\": (\n", + " \"Y coordinate of the supraglacial lake polygon centroid. For Antarctic geographic \"\n", + " \"datasets this is typically latitude in decimal degrees, unless the source layer \"\n", + " \"uses a projected coordinate reference system.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"Area\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Supraglacial lake area\",\n", + " \"description\": (\n", + " \"Surface area of the mapped supraglacial lake feature. Used to describe lake size \"\n", + " \"and to support derived metrics such as volume estimates and area-perimeter ratios.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"Shape_Leng\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake polygon perimeter\",\n", + " \"description\": (\n", + " \"Perimeter or boundary length of the supraglacial lake polygon geometry.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Shape_Area\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake polygon shape area\",\n", + " \"description\": (\n", + " \"Area of the lake polygon geometry calculated by the source GIS software. This may \"\n", + " \"duplicate or closely correspond to POLY_AREA or Area.\"\n", + " ),\n", + " \"unit\": \"m2\",\n", + " },\n", + " {\n", + " \"name\": \"Bedrock\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to exposed bedrock\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the nearest mapped exposed bedrock \"\n", + " \"or rock outcrop, where available.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Coast\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to coast\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the Antarctic coastline or ice-shelf \"\n", + " \"front used in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"GL_ON_IS\",\n", + " \"type\": \"boolean\",\n", + " \"long_name\": \"Grounding-line-on-ice-shelf flag\",\n", + " \"description\": (\n", + " \"Flag indicating whether the supraglacial lake is located on an ice shelf or in a \"\n", + " \"grounding-line-related ice-shelf setting, according to the source classification.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"Location\",\n", + " \"type\": \"string\",\n", + " \"long_name\": \"Lake location class\",\n", + " \"description\": (\n", + " \"Textual location category or regional setting assigned to the supraglacial lake, \"\n", + " \"such as ice sheet, ice shelf, grounding-zone, coastal, or named regional grouping, \"\n", + " \"depending on the source classification.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"GL\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Distance to grounding line\",\n", + " \"description\": (\n", + " \"Distance from the supraglacial lake feature to the Antarctic grounding line used \"\n", + " \"in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Volume\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Estimated lake volume\",\n", + " \"description\": (\n", + " \"Estimated water volume of the supraglacial lake feature, typically derived from \"\n", + " \"mapped lake area and an empirical or remotely sensed depth relationship.\"\n", + " ),\n", + " \"unit\": \"m3\",\n", + " },\n", + " {\n", + " \"name\": \"A_P_Ratio\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Area-perimeter ratio\",\n", + " \"description\": (\n", + " \"Ratio of lake polygon area to perimeter, used as a compactness or shape metric for \"\n", + " \"the mapped supraglacial lake.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"REMA_Max\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Maximum REMA elevation\",\n", + " \"description\": (\n", + " \"Maximum surface elevation from the Reference Elevation Model of Antarctica within \"\n", + " \"or associated with the supraglacial lake polygon.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"REMA_Min\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Minimum REMA elevation\",\n", + " \"description\": (\n", + " \"Minimum surface elevation from the Reference Elevation Model of Antarctica within \"\n", + " \"or associated with the supraglacial lake polygon.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"PolsbyPopp\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Polsby-Popper compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric for the lake polygon based on the ratio of polygon area \"\n", + " \"to the area of a circle with the same perimeter. Values closer to 1 indicate a more \"\n", + " \"compact, circular shape.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Fractal\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Fractal dimension shape index\",\n", + " \"description\": (\n", + " \"Shape-complexity metric describing the irregularity of the supraglacial lake polygon \"\n", + " \"boundary. Higher values generally indicate a more complex or convoluted outline.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Reock\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Reock compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric comparing the polygon area with the area of its minimum \"\n", + " \"bounding circle. Values closer to 1 indicate a more compact shape.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Schwartzbe\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Schwartzberg compactness\",\n", + " \"description\": (\n", + " \"Shape compactness metric comparing the polygon perimeter with the circumference of \"\n", + " \"a circle of equal area. Values closer to 1 generally indicate a more compact, \"\n", + " \"circular polygon.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"LW_Ratio\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Length-width ratio\",\n", + " \"description\": (\n", + " \"Ratio of lake length to lake width, used to describe elongation of the supraglacial \"\n", + " \"lake polygon.\"\n", + " ),\n", + " \"unit\": \"1\",\n", + " },\n", + " {\n", + " \"name\": \"Feature_Cl\",\n", + " \"type\": \"string\",\n", + " \"long_name\": \"Feature class\",\n", + " \"description\": (\n", + " \"Classification assigned to the mapped feature in the source dataset. For this asset, \"\n", + " \"features represent supraglacial lakes, but this field may preserve source-level \"\n", + " \"feature class labels or quality classes.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + " {\n", + " \"name\": \"Elevation\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Lake surface elevation\",\n", + " \"description\": (\n", + " \"Representative surface elevation associated with the supraglacial lake feature, \"\n", + " \"likely derived from REMA or another Antarctic digital elevation model.\"\n", + " ),\n", + " \"unit\": \"m\",\n", + " },\n", + " {\n", + " \"name\": \"Slope\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Surface slope\",\n", + " \"description\": (\n", + " \"Representative ice-surface slope associated with the supraglacial lake feature.\"\n", + " ),\n", + " \"unit\": \"degree\",\n", + " },\n", + " {\n", + " \"name\": \"Speed\",\n", + " \"type\": \"float\",\n", + " \"long_name\": \"Ice surface velocity\",\n", + " \"description\": (\n", + " \"Representative ice-flow speed at or near the supraglacial lake feature, derived from \"\n", + " \"an Antarctic ice-velocity product used in the source analysis.\"\n", + " ),\n", + " \"unit\": \"m yr-1\",\n", + " },\n", + " {\n", + " \"name\": \"geometry\",\n", + " \"type\": \"geometry\",\n", + " \"long_name\": \"Supraglacial lake polygon geometry\",\n", + " \"description\": (\n", + " \"Polygon geometry representing the mapped supraglacial lake surface extent. Stored as \"\n", + " \"the GeoParquet geometry column.\"\n", + " ),\n", + " \"unit\": None,\n", + " },\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f10cb211", + "metadata": {}, + "outputs": [], + "source": [ + "item.add_asset('data', asset)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2510d5c3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=supraglacial-lakes-boundries-only>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.validate()\n", + "item" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2060df3c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ff4282e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9a84ecc8", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from pathlib import Path\n", + "from earthcode.static import generic_stac_item_to_earthcode_stac_item\n", + "from earthcode.git_add import save_item_to_product_collection\n", + "\n", + "# specify catalog root and OSC collection\n", + "catalog_root = Path('../../../open-science-catalog-metadata/')\n", + "collection = 'sofresh-sea-surface-salinity'" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "50a879b0", + "metadata": {}, + "outputs": [], + "source": [ + "# convert to earthcode item with a reference to a specific open science catalog collection\n", + "earthcode_item = generic_stac_item_to_earthcode_stac_item(item, collection, asset_key='data')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e9bab85d", + "metadata": {}, + "outputs": [], + "source": [ + "# save the item to the collection\n", + "save_item_to_product_collection(earthcode_item, collection, catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "224378a0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "270da193", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([], [])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# validate catalog\n", + "from earthcode.validator import validate_catalog\n", + "validate_catalog(catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32fe2577", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/zarr_file_metadata.ipynb b/docs/tutorials/zarr_file_metadata.ipynb new file mode 100644 index 0000000..3bfabe6 --- /dev/null +++ b/docs/tutorials/zarr_file_metadata.ipynb @@ -0,0 +1,6113 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c4ba7099", + "metadata": {}, + "source": [ + "# Zarr Metadata With xstac\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ccc655b1", + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "from rasterio.warp import transform_bounds\n", + "import rioxarray\n", + "from xstac import xarray_to_stac\n", + "import pandas as pd\n", + "import requests\n", + "from shapely.geometry import mapping, box" + ] + }, + { + "cell_type": "markdown", + "id": "954c71bf", + "metadata": {}, + "source": [ + "## Open The Remote Zarr Store and Attach CRS Metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "13dc259a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 18GB\n",
+       "Dimensions:          (time: 4441, y: 720, x: 720, nv: 2)\n",
+       "Coordinates:\n",
+       "  * time             (time) datetime64[ns] 36kB 2011-02-01 ... 2023-03-30T23:...\n",
+       "  * y                (y) float32 3kB 8.988e+06 8.962e+06 ... -8.988e+06\n",
+       "  * x                (x) float32 3kB -8.988e+06 -8.962e+06 ... 8.988e+06\n",
+       "    lat              (y, x) float32 2MB dask.array<chunksize=(720, 720), meta=np.ndarray>\n",
+       "    lon              (y, x) float32 2MB dask.array<chunksize=(720, 720), meta=np.ndarray>\n",
+       "Dimensions without coordinates: nv\n",
+       "Data variables:\n",
+       "    crs              int32 4B ...\n",
+       "    sss              (time, y, x) float32 9GB dask.array<chunksize=(10, 720, 720), meta=np.ndarray>\n",
+       "    time_bnds        (time, nv) datetime64[ns] 71kB dask.array<chunksize=(4441, 2), meta=np.ndarray>\n",
+       "    uncertainty_sss  (time, y, x) float32 9GB dask.array<chunksize=(10, 720, 720), meta=np.ndarray>\n",
+       "Attributes: (12/35)\n",
+       "    date_created:             2023-07-12 06:14:00 GMT\n",
+       "    geospatial_x_units:       m\n",
+       "    geospatial_y_units:       m\n",
+       "    geospatial_x_resolution:  25000.0\n",
+       "    geospatial_y_resolution:  25000.0\n",
+       "    geospatial_x_min:         -9000000.0\n",
+       "    ...                       ...\n",
+       "    copyright:                If this data is used for publication, the follo...\n",
+       "    comment:                  This data was produced at BEC as part of the So...\n",
+       "    time_coverage_start:      20110201T00:00:00\n",
+       "    time_coverage_end:        1680217200000000000\n",
+       "    DOI:                      10.20350/digitalCSIC/15493\n",
+       "    title:                    Southern Ocean SMOS Level 3 Sea Surface Salinit...
" + ], + "text/plain": [ + " Size: 18GB\n", + "Dimensions: (time: 4441, y: 720, x: 720, nv: 2)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 36kB 2011-02-01 ... 2023-03-30T23:...\n", + " * y (y) float32 3kB 8.988e+06 8.962e+06 ... -8.988e+06\n", + " * x (x) float32 3kB -8.988e+06 -8.962e+06 ... 8.988e+06\n", + " lat (y, x) float32 2MB dask.array\n", + " lon (y, x) float32 2MB dask.array\n", + "Dimensions without coordinates: nv\n", + "Data variables:\n", + " crs int32 4B ...\n", + " sss (time, y, x) float32 9GB dask.array\n", + " time_bnds (time, nv) datetime64[ns] 71kB dask.array\n", + " uncertainty_sss (time, y, x) float32 9GB dask.array\n", + "Attributes: (12/35)\n", + " date_created: 2023-07-12 06:14:00 GMT\n", + " geospatial_x_units: m\n", + " geospatial_y_units: m\n", + " geospatial_x_resolution: 25000.0\n", + " geospatial_y_resolution: 25000.0\n", + " geospatial_x_min: -9000000.0\n", + " ... ...\n", + " copyright: If this data is used for publication, the follo...\n", + " comment: This data was produced at BEC as part of the So...\n", + " time_coverage_start: 20110201T00:00:00\n", + " time_coverage_end: 1680217200000000000\n", + " DOI: 10.20350/digitalCSIC/15493\n", + " title: Southern Ocean SMOS Level 3 Sea Surface Salinit..." + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "collection = 'sofresh-sea-surface-salinity'\n", + "zarr_href = 'https://s3.waw4-1.cloudferro.com/EarthCODE/OSCAssets/polar_cube_datasets/so-fresh/sofresh.zarr/'\n", + "ds = xr.open_zarr(zarr_href)\n", + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f0280a37", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 18GB\n",
+       "Dimensions:          (time: 4441, y: 720, x: 720, nv: 2)\n",
+       "Coordinates:\n",
+       "  * time             (time) datetime64[ns] 36kB 2011-02-01 ... 2023-03-30T23:...\n",
+       "  * y                (y) float32 3kB 8.988e+06 8.962e+06 ... -8.988e+06\n",
+       "  * x                (x) float32 3kB -8.988e+06 -8.962e+06 ... 8.988e+06\n",
+       "    crs              int64 8B 0\n",
+       "    lat              (y, x) float32 2MB dask.array<chunksize=(720, 720), meta=np.ndarray>\n",
+       "    lon              (y, x) float32 2MB dask.array<chunksize=(720, 720), meta=np.ndarray>\n",
+       "Dimensions without coordinates: nv\n",
+       "Data variables:\n",
+       "    sss              (time, y, x) float32 9GB dask.array<chunksize=(10, 720, 720), meta=np.ndarray>\n",
+       "    time_bnds        (time, nv) datetime64[ns] 71kB dask.array<chunksize=(4441, 2), meta=np.ndarray>\n",
+       "    uncertainty_sss  (time, y, x) float32 9GB dask.array<chunksize=(10, 720, 720), meta=np.ndarray>\n",
+       "Attributes: (12/35)\n",
+       "    date_created:             2023-07-12 06:14:00 GMT\n",
+       "    geospatial_x_units:       m\n",
+       "    geospatial_y_units:       m\n",
+       "    geospatial_x_resolution:  25000.0\n",
+       "    geospatial_y_resolution:  25000.0\n",
+       "    geospatial_x_min:         -9000000.0\n",
+       "    ...                       ...\n",
+       "    copyright:                If this data is used for publication, the follo...\n",
+       "    comment:                  This data was produced at BEC as part of the So...\n",
+       "    time_coverage_start:      20110201T00:00:00\n",
+       "    time_coverage_end:        1680217200000000000\n",
+       "    DOI:                      10.20350/digitalCSIC/15493\n",
+       "    title:                    Southern Ocean SMOS Level 3 Sea Surface Salinit...
" + ], + "text/plain": [ + " Size: 18GB\n", + "Dimensions: (time: 4441, y: 720, x: 720, nv: 2)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 36kB 2011-02-01 ... 2023-03-30T23:...\n", + " * y (y) float32 3kB 8.988e+06 8.962e+06 ... -8.988e+06\n", + " * x (x) float32 3kB -8.988e+06 -8.962e+06 ... 8.988e+06\n", + " crs int64 8B 0\n", + " lat (y, x) float32 2MB dask.array\n", + " lon (y, x) float32 2MB dask.array\n", + "Dimensions without coordinates: nv\n", + "Data variables:\n", + " sss (time, y, x) float32 9GB dask.array\n", + " time_bnds (time, nv) datetime64[ns] 71kB dask.array\n", + " uncertainty_sss (time, y, x) float32 9GB dask.array\n", + "Attributes: (12/35)\n", + " date_created: 2023-07-12 06:14:00 GMT\n", + " geospatial_x_units: m\n", + " geospatial_y_units: m\n", + " geospatial_x_resolution: 25000.0\n", + " geospatial_y_resolution: 25000.0\n", + " geospatial_x_min: -9000000.0\n", + " ... ...\n", + " copyright: If this data is used for publication, the follo...\n", + " comment: This data was produced at BEC as part of the So...\n", + " time_coverage_start: 20110201T00:00:00\n", + " time_coverage_end: 1680217200000000000\n", + " DOI: 10.20350/digitalCSIC/15493\n", + " title: Southern Ocean SMOS Level 3 Sea Surface Salinit..." + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = ds.rio.write_crs('ESRI:102020')\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "7c75c569", + "metadata": {}, + "source": [ + "## Derive Spatial And Temporal Extents\n", + "\n", + "This cell reads the projected `x` and `y` coordinate limits, transforms them to WGS84, then sets the final geographic bounding box used in STAC. It also converts the dataset time coordinate range into Python datetimes.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7c5451ca", + "metadata": {}, + "outputs": [], + "source": [ + "minx, maxx, miny, maxy = ds.x.min().values, ds.x.max().values, ds.y.min().values, ds.y.max().values\n", + "\n", + "minx = float(ds.x.min())\n", + "maxx = float(ds.x.max())\n", + "miny = float(ds.y.min())\n", + "maxy = float(ds.y.max())\n", + "\n", + "minx_4326, miny_4326, maxx_4326, maxy_4326 = transform_bounds(\n", + " ds.rio.crs, \"EPSG:4326\", minx, miny, maxx, maxy, densify_pts=101\n", + ")\n", + "\n", + "min_time, max_time = ds.time.values.ravel().min(), ds.time.values.ravel().max()\n", + "\n", + "\n", + "minx_4326, miny_4326, maxx_4326, maxy_4326 = (-180.0, -90.0, 180.0, -56.7587107166777)\n", + "start = pd.Timestamp(min_time).tz_localize(\"UTC\").to_pydatetime()\n", + "end = pd.Timestamp(max_time).tz_localize(\"UTC\").to_pydatetime()" + ] + }, + { + "cell_type": "markdown", + "id": "3bb2ba98", + "metadata": {}, + "source": [ + "## Build The STAC Template And Let xstac Extract Metadata\n", + "\n", + "The template supplies item-level fields such as ID, licence, temporal extent, geometry, and the Zarr data asset. `xarray_to_stac` then combines that template with metadata extracted from the xarray dataset. The final `item.validate()` call checks the generated STAC object with pystac validation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1d45ba35", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_86034/710910711.py:1: Pandas4Warning: Timestamp.utcnow is deprecated and will be removed in a future version. Use Timestamp.now('UTC') instead.\n", + " raw_created = ds.attrs.get(\"product_created\", ds.attrs.get(\"date_modified\", pd.Timestamp.utcnow()))\n" + ] + } + ], + "source": [ + "raw_created = ds.attrs.get(\"product_created\", ds.attrs.get(\"date_modified\", pd.Timestamp.utcnow()))\n", + "created_utc = pd.to_datetime(raw_created, utc=True).isoformat().replace(\"+00:00\", \"Z\")\n", + "\n", + "start_utc = pd.to_datetime(start, utc=True).isoformat().replace(\"+00:00\", \"Z\")\n", + "end_utc = pd.to_datetime(end, utc=True).isoformat().replace(\"+00:00\", \"Z\")\n", + "\n", + "item_id = f\"{collection}-item\"\n", + "\n", + "template = {\n", + " \"id\": item_id,\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"properties\": {\n", + " \"title\": f\"Sea surface salinity zarr store\",\n", + " \"start_datetime\": start_utc,\n", + " \"end_datetime\": end_utc,\n", + " \"license\": 'cc-by-4.0',\n", + " \"created\": created_utc,\n", + " },\n", + " \"bbox\": [minx_4326, miny_4326, maxx_4326, maxy_4326],\n", + " \"geometry\": mapping(box(minx_4326, miny_4326, maxx_4326, maxy_4326)),\n", + " \"assets\": {\n", + " \"data\": {\n", + " \"href\": zarr_href,\n", + " \"type\": \"application/vnd+zarr\",\n", + " \"roles\": [\"data\"],\n", + " \"title\": \"Zarr Store\",\n", + " }\n", + " },\n", + "}\n", + "\n", + "item = xarray_to_stac(\n", + " ds,\n", + " template,\n", + " temporal_dimension=\"time\",\n", + " x_dimension=\"x\",\n", + " y_dimension=\"y\",\n", + " validate=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "de7fe15e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/item.json',\n", + " 'https://stac-extensions.github.io/datacube/v2.2.0/schema.json']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "### bypass proj http block by m,anually downloading the schema\n", + "from pystac.validation import RegisteredValidator\n", + "\n", + "url = 'https://proj.org/en/latest/schemas/v0.4/projjson.schema.json'\n", + "response = requests.get(url)\n", + "response.raise_for_status()\n", + "schema_data = response.json()\n", + "\n", + "validator = RegisteredValidator.get_validator()\n", + "schema_url = \"https://proj.org/schemas/v0.4/projjson.schema.json\"\n", + "validator.schema_cache[schema_url] = schema_data\n", + "\n", + "\n", + "item.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53f065a2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "99048154", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from pathlib import Path\n", + "from earthcode.static import generic_stac_item_to_earthcode_stac_item\n", + "from earthcode.git_add import save_item_to_product_collection\n", + "\n", + "catalog_root = Path('../../../open-science-catalog-metadata/')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d542a180", + "metadata": {}, + "outputs": [], + "source": [ + "# convert to earthcode item with a reference to a specific open science catalog collection\n", + "earthcode_item = generic_stac_item_to_earthcode_stac_item(item, collection, asset_key='data')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9adf6ba9", + "metadata": {}, + "outputs": [], + "source": [ + "# save the item to the collection\n", + "save_item_to_product_collection(earthcode_item, collection, catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cda258c4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([], [])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# validate catalog\n", + "from earthcode.validator import validate_catalog\n", + "validate_catalog(catalog_root)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "26f4047e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + "
\n", + " <Item id=sofresh-sea-surface-salinity-item>\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "earthcode_item" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7502d8b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "default", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/earthcode/static.py b/earthcode/static.py index 22ba76a..6a1863b 100644 --- a/earthcode/static.py +++ b/earthcode/static.py @@ -3,6 +3,8 @@ from datetime import datetime, timezone import pystac +import json +import shapely from pystac.extensions.scientific import ScientificExtension from earthcode.metadata_input_definitions import ( @@ -143,7 +145,12 @@ def _apply_project_contacts( _ensure_extension(collection, CONTACTS_SCHEMA_URI) to_contact = _create_contact(technical_officer_name, ["technical_officer"], [technical_officer_email]) - consortium_contacts = [_create_contact(name, ["consoritum_member"], [email]) for name, email in consortium_members] + consortium_contacts = [] + for name, email in consortium_members: + if email and len(email) > 0: + consortium_contacts.append(_create_contact(name, ["consoritum_member"], [email])) + else: + consortium_contacts.append(_create_contact(name, ["consoritum_member"], None)) collection.extra_fields["contacts"] = [to_contact] + consortium_contacts @@ -569,4 +576,72 @@ def create_item(item_metadata: ItemMetadata, stac_version='1.0.0') -> pystac.Ite ), ) - return item \ No newline at end of file + return item + +def generic_stac_item_to_earthcode_stac_item(item: pystac.Item, collection:str, asset_key: str = None): + + # 1. Core Item Metadata + item_id = item.id + item_bbox = item.bbox if item.bbox else [-180.0, -90.0, 180.0, 90.0] + + # Handle datetime (STAC items might use start/end instead of a single datetime) + if item.datetime: + item_datetime = item.datetime.isoformat() + else: + item_datetime = item.properties.get("start_datetime", "") + + # STAC generally stores license/description in the Collection, but they can + # exist in the Item's properties. + item_license = item.properties.get("license", "unknown") + item_description = item.properties.get("description", "") + + # 2. Asset Metadata + item_data_url = "" + item_data_mime_type = "" + item_data_title = item.properties['title'] if 'title' in item.properties else "" + + target_asset = None + if asset_key and asset_key in item.assets: + target_asset = item.assets[asset_key] + elif item.assets: + # Fallback to the first asset in the dictionary if no key is provided + target_asset = list(item.assets.values())[0] + + if target_asset: + item_data_url = target_asset.href + item_data_mime_type = target_asset.media_type or "" + item_data_title = target_asset.title or item_data_title + + # 3. Extra Fields + # Filter out properties we already explicitly assigned to avoid duplication + known_keys = {"datetime", "start_datetime", "end_datetime", "license", "description"} + item_extra_fields = { + key: value + for key, value in item.properties.items() + if key not in known_keys + } + + ## generate a STAC item associated with a specific collection + geometry = json.loads(json.dumps(shapely.box(*item_bbox).__geo_interface__)) + item_metadata = ItemMetadata( + itemid=item_id, + geometry=geometry, + data_time=item_datetime, + bbox=item_bbox, + product_id=collection, + license=item_license, + description=item_description, + data_url=item_data_url, + data_mime_type=item_data_mime_type, + data_title=item_data_title, + extra_fields=item_extra_fields, + ) + earthcode_item = create_item(item_metadata) + + if 'start_datetime' in item.properties: + earthcode_item.properties['start_datetime'] = item.properties['start_datetime'] + earthcode_item.properties['end_datetime'] = item.properties['end_datetime'] + earthcode_item.properties['datetime'] = item.properties['datetime'] + + earthcode_item.extra_fields['title'] = item_data_title + return earthcode_item \ No newline at end of file diff --git a/examples/contribute_via_osc_editor.ipynb b/examples/contribute_via_osc_editor.ipynb deleted file mode 100644 index 2c61dd6..0000000 --- a/examples/contribute_via_osc_editor.ipynb +++ /dev/null @@ -1,2003 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5655dd7c", - "metadata": {}, - "source": [ - "# Contribute to Open Science Catalogue via the OSC editor\n", - "\n", - "This notebook is an example of how to add a new entry - i.e. `product, project, workflow` - to the Open Science Catalogue (OSC) via a combination of the `earthcode` library and the [Open Science Catalog Editor](https://esa-earthcode.github.io/tutorials/git-clerk-example/).\n", - "\n", - "The process to do this is three-fold:\n", - "1. Create the OSC entries using the library\n", - "2. Generate links and send the data to the OSC Editor\n", - "3. Open Pull Request via the OSC for your project entry\n", - "4. Open Pull Request via the OSC for your product entry\n", - "\n", - "Before starting, please checkout the introduction to the [Open Science Catalog Editor](https://esa-earthcode.github.io/tutorials/git-clerk-example/) if you are not familiar with it already.\n", - "\n", - "\n", - "#### 1. Create the OSC entries using the library\n", - "In this notebook we use a dummy project/product pair for demonstration purposes. You can reuse the code below using real OSC entries.
" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "cf0c8b79", - "metadata": {}, - "outputs": [], - "source": [ - "from earthcode.static import generate_OSC_dummy_entries" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "a6f96ef3", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "
\n", - "
\n", - " <Collection id=4datlantic-ohc+123>\n", - "
\n", - "\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "project_collection, product_collection, _, _ = generate_OSC_dummy_entries()\n", - "project_collection" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "35ffabe0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "
\n", - "
\n", - " <Collection id=4d-atlantic-ohc-global+123>\n", - "
\n", - "\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "product_collection" - ] - }, - { - "cell_type": "markdown", - "id": "4bc49e3d", - "metadata": {}, - "source": [ - "## 2. Generate links and send the data to the OSC Editor\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1eed3b9c", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Copy the the below link in a webbrowser to setup the project information: \n", - "https://workspace.earthcode-staging.earthcode.eox.at/osc-editor?session=4DAtlantic-OHC&automation=add-file&&type=project&file=eyJ0eXBlIjogIkNvbGxlY3Rpb24iLCAiaWQiOiAiNGRhdGxhbnRpYy1vaGMrMTIzIiwgInN0YWNfdmVyc2lvbiI6ICIxLjAuMCIsICJkZXNjcmlwdGlvbiI6ICJHaXZlbiB0aGUgbWFqb3Igcm9sZSBvZiB0aGUgb2NlYW4gaW4gdGhlIGNsaW1hdGUgc3lzdGVtLCBpdCBpcyBlc3NlbnRpYWwgdG8gY2hhcmFjdGVyaXplIHRoZSB0ZW1wb3JhbCBhbmQgc3BhdGlhbCB2YXJpYXRpb25zIG9mIGl0cyBoZWF0IGNvbnRlbnQuIFRoZSBPSEMgcHJvZHVjdCByZXN1bHRzIGZyb20gdGhlIHNwYWNlIGdlb2RldGljIGFwcHJvYWNoIGFsc28gY2FsbGVkIGFsdGltZXRyeS1ncmF2aW1ldHJ5IGFwcHJvYWNoLiIsICJsaW5rcyI6IFt7InJlbCI6ICJyb290IiwgImhyZWYiOiAiLi4vLi4vY2F0YWxvZy5qc29uIiwgInR5cGUiOiAiYXBwbGljYXRpb24vanNvbiIsICJ0aXRsZSI6ICJPcGVuIFNjaWVuY2UgQ2F0YWxvZyJ9LCB7InJlbCI6ICJwYXJlbnQiLCAiaHJlZiI6ICIuLi9jYXRhbG9nLmpzb24iLCAidHlwZSI6ICJhcHBsaWNhdGlvbi9qc29uIiwgInRpdGxlIjogIlByb2plY3RzIn0sIHsicmVsIjogInZpYSIsICJocmVmIjogImh0dHBzOi8vd3d3LjRkYXRsYW50aWMtb2hjLm9yZy8iLCAidGl0bGUiOiAiV2Vic2l0ZSJ9LCB7InJlbCI6ICJ2aWEiLCAiaHJlZiI6ICJodHRwczovL2VvNHNvY2lldHkuZXNhLmludC9wcm9qZWN0cy80ZGF0bGFudGljLW9oYy8iLCAidGl0bGUiOiAiRU80U29jaWV0eSBMaW5rIn0sIHsicmVsIjogInJlbGF0ZWQiLCAiaHJlZiI6ICIuLi8uLi90aGVtZXMvb2NlYW5zL2NhdGFsb2cuanNvbiIsICJ0eXBlIjogImFwcGxpY2F0aW9uL2pzb24iLCAidGl0bGUiOiAiVGhlbWU6IE9jZWFucyJ9XSwgInN0YWNfZXh0ZW5zaW9ucyI6IFsiaHR0cHM6Ly9zdGFjLWV4dGVuc2lvbnMuZ2l0aHViLmlvL29zYy92MS4wLjAvc2NoZW1hLmpzb24iLCAiaHR0cHM6Ly9zdGFjLWV4dGVuc2lvbnMuZ2l0aHViLmlvL3RoZW1lcy92MS4wLjAvc2NoZW1hLmpzb24iLCAiaHR0cHM6Ly9zdGFjLWV4dGVuc2lvbnMuZ2l0aHViLmlvL2NvbnRhY3RzL3YwLjEuMS9zY2hlbWEuanNvbiJdLCAib3NjOnN0YXR1cyI6ICJjb21wbGV0ZWQiLCAib3NjOnR5cGUiOiAicHJvamVjdCIsICJ1cGRhdGVkIjogIjIwMjYtMDItMjVUMTM6MzY6MjRaIiwgInRoZW1lcyI6IFt7InNjaGVtZSI6ICJodHRwczovL2dpdGh1Yi5jb20vc3RhYy1leHRlbnNpb25zL29zYyN0aGVtZSIsICJjb25jZXB0cyI6IFt7ImlkIjogIm9jZWFucyJ9XX1dLCAiY29udGFjdHMiOiBbeyJuYW1lIjogIlJvYmVydG8gU2FiaWEiLCAicm9sZXMiOiBbInRlY2huaWNhbF9vZmZpY2VyIl0sICJlbWFpbHMiOiBbeyJ2YWx1ZSI6ICJyb2JlcnRvLnNhYmlhQGVzYS5pbnQifV19LCB7Im5hbWUiOiAiTWFnZWxsaXVtIiwgInJvbGVzIjogWyJjb25zb3JpdHVtX21lbWJlciJdLCAiZW1haWxzIjogW3sidmFsdWUiOiAibWFnZWxsaXVtLmZyIn1dfV0sICJ0aXRsZSI6ICI0REF0bGFudGljLU9IQyIsICJleHRlbnQiOiB7InNwYXRpYWwiOiB7ImJib3giOiBbWy0xODAuMCwgLTkwLjAsIDE4MC4wLCA5MC4wXV19LCAidGVtcG9yYWwiOiB7ImludGVydmFsIjogW1siMjAyMS0wNy0wNlQwMDowMDowMFoiLCAiMjAyNS0wNi0xMlQwMDowMDowMFoiXV19fSwgImxpY2Vuc2UiOiAidmFyaW91cyJ9\n" - ] - } - ], - "source": [ - "from earthcode.gitclerk_add import generate_osc_editor_link\n", - "\n", - "# NOTE: Always save the project first\n", - "project_url = generate_osc_editor_link(project_collection, 'project', session_title=project_collection.title)\n", - "print('Copy the the below link in a webbrowser to setup the project information: ')\n", - "print(project_url)" - ] - }, - { - "cell_type": "markdown", - "id": "b135e726", - "metadata": {}, - "source": [ - "### 3. Open a PR via the OSC Editor\n", - "\n", - "Now you can use the OSC Editor and GUI to preview your data and open a pull request.\n", - "\n", - "\n", - "After creation of Pull Request you should see it on the list: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/pulls\n", - "\n", - "Check the status of your PR under: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/actions\n", - "\n", - "> Changes to the OSC content will be reviewed by the OSC Data Steward team. In case of any chenges needed to your inputs, you will be contacted by the team." - ] - }, - { - "cell_type": "markdown", - "id": "ab597922", - "metadata": {}, - "source": [ - "## 4. Repeat for product or workflow.\n", - "\n", - "Once the project is added to the OSC, you can repeat the same process for your product." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df70d47a", - "metadata": {}, - "outputs": [], - "source": [ - "# Now save the product data\n", - "# NOTE: Always use the same sesion title for project,product or project, workflow pairs, etc\n", - "product_url = generate_osc_editor_link(product_collection, 'product', session_title=project_collection.title)\n", - "print('Click on the below link to setup the product information: ')\n", - "print(product_url)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/contribute_via_pr_osc.ipynb b/examples/contribute_via_pr_osc.ipynb deleted file mode 100644 index 24d2f6d..0000000 --- a/examples/contribute_via_pr_osc.ipynb +++ /dev/null @@ -1,2278 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5655dd7c", - "metadata": {}, - "source": [ - "# Contribute to Open Science Catalogue with Pull Request (GitHub)\n", - "\n", - "This notebook is an example of how to add a new entry - i.e. `product, project, workflow, experiment` - to the Open Science Catalogue (OSC) via GitHub Pull Request. \n", - "This can be done using [Graphical User Interface](https://esa-earthcode.github.io/tutorials/git-clerk-example/) within [EarthCODE workspace](https://workspace.earthcode.eox.at/), manual creation of the entries using web browser on GitHub, or by using platforms-specific tools. \n", - "This notebook covers steps on how publishing can be done, locally, wothout a need of sepcific platform or installation of the additional tools.
This document covers following steps:\n", - "1. Forking a local copy of the OSC\n", - "2. Embedding new OSC entries into the Catalog\n", - "3. Validation\n", - "4. Open Pull Request to add new entries\n", - "5. What is next?
\n", - "\n", - "If you have your OSC already prepared please go directly to **`Step 1.`** to see how you can add your inputs into Open Science Catalogue.
\n", - "\n", - "> To proceed with this notebook, **you need an active GitHub account**. If you do not have one, please [create an account](https://github.com/signup) to get started first. \n", - "\n", - "#### 0. Load dummy OSC entries\n", - "In this notebook we use a group of dummy project, product, workflow and exeriment data for demonstration purposes. You can reuse the code below using real OSC entries.
\n", - "\n", - "**NOTE** If you do not have any entries ready, you can use following (testing example) to see whether the publishing schema proposed works on your environment. Once you are familiar with these steps, you can generate valid OSC entries for your specific project!
\n", - "Run cells 1-5 to preview the *OSC dummy entries* prepared only for demonstration purposes! " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "cf0c8b79", - "metadata": {}, - "outputs": [], - "source": [ - "from earthcode.static import generate_OSC_dummy_entries" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "a6f96ef3", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "
\n", - "
\n", - " <Collection id=4datlantic-ohc+123>\n", - "
\n", - "\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "project_collection, product_collection, workflow_record, experiment_record = generate_OSC_dummy_entries()\n", - "project_collection" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "35ffabe0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "
\n", - "
\n", - " <Collection id=4d-atlantic-ohc-global+123>\n", - "
\n", - "\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "product_collection" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "4b3ae24f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': '4datlantic-wf+123',\n", - " 'type': 'Feature',\n", - " 'geometry': None,\n", - " 'conformsTo': ['http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core'],\n", - " 'properties': {'title': '4D-Atlantic-Workflow',\n", - " 'description': 'This describes the OHC workflow',\n", - " 'osc:type': 'workflow',\n", - " 'osc:project': '4datlantic-ohc',\n", - " 'osc:status': 'completed',\n", - " 'formats': [{'name': 'netcdf64'}],\n", - " 'updated': '2026-01-27T16:38:01Z',\n", - " 'created': '2026-01-27T16:38:01Z',\n", - " 'keywords': ['ocean', 'heat', 'çontent'],\n", - " 'license': 'CC-BY-4.0',\n", - " 'version': '1',\n", - " 'themes': [{'scheme': 'https://github.com/stac-extensions/osc#theme',\n", - " 'concepts': [{'id': 'oceans'}]}]},\n", - " 'linkTemplates': [],\n", - " 'links': [{'rel': 'root',\n", - " 'href': '../../catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Open Science Catalog'},\n", - " {'rel': 'parent',\n", - " 'href': '../catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Workflows'},\n", - " {'rel': 'related',\n", - " 'href': '../../projects/4D Atlantic OHC/collection.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Project: 4D Atlantic OHC'},\n", - " {'rel': 'git',\n", - " 'href': 'https://github.com/ESA-EarthCODE/open-science-catalog-metadata',\n", - " 'type': 'application/json',\n", - " 'title': 'Git source repository'},\n", - " {'rel': 'related',\n", - " 'href': '../../oceans/land/catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Theme: Oceans'}]}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "workflow_record" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "752ac44d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': '4datlantic-experiment+123',\n", - " 'type': 'Feature',\n", - " 'conformsTo': ['http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core'],\n", - " 'geometry': None,\n", - " 'properties': {'created': '2026-01-27T16:38:01Z',\n", - " 'updated': '2026-01-27T16:38:01Z',\n", - " 'type': 'experiment',\n", - " 'title': '4D-Atlantic-Experiment',\n", - " 'description': 'This describes the OHC experiment',\n", - " 'keywords': ['ocean', 'heat', 'content'],\n", - " 'contacts': [{'name': 'EarthCODE Demo',\n", - " 'organization': 'EarthCODE',\n", - " 'links': [{'rel': 'about',\n", - " 'type': 'text/html',\n", - " 'href': 'https://opensciencedata.esa.int/'}],\n", - " 'contactInstructions': 'Contact via EarthCODE',\n", - " 'roles': ['host']}],\n", - " 'themes': [{'scheme': 'https://github.com/stac-extensions/osc#theme',\n", - " 'concepts': [{'id': 'oceans'}]}],\n", - " 'formats': [{'name': 'GeoTIFF'}],\n", - " 'license': 'CC-BY-SA-4.0',\n", - " 'osc:workflow': '4datlantic-wf+123'},\n", - " 'linkTemplates': [],\n", - " 'links': [{'rel': 'root',\n", - " 'href': '../../catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Open Science Catalog'},\n", - " {'rel': 'parent',\n", - " 'href': '../catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Experiments'},\n", - " {'rel': 'related',\n", - " 'href': '../../products/4d-atlantic-ohc-global+123/collection.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Global Ocean Heat Content'},\n", - " {'rel': 'related',\n", - " 'href': '../../workflows/4datlantic-wf+123/record.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Workflow: 4D-Atlantic-Workflow'},\n", - " {'rel': 'input',\n", - " 'href': 'https://github.com/deepesdl/cube-gen',\n", - " 'type': 'application/yaml',\n", - " 'title': 'Input parameters'},\n", - " {'rel': 'environment',\n", - " 'href': 'https://github.com/deepesdl/cube-gen',\n", - " 'type': 'application/yaml',\n", - " 'title': 'Execution environment'},\n", - " {'rel': 'related',\n", - " 'href': '../../themes/oceans/catalog.json',\n", - " 'type': 'application/json',\n", - " 'title': 'Theme: Oceans'}]}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "experiment_record" - ] - }, - { - "cell_type": "markdown", - "id": "4bc49e3d", - "metadata": {}, - "source": [ - "## 1. Setup a local Copy of the OSC\n", - "\n", - "\n", - "You can add new content to the OSC via GitHub [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). To do this, you need a to [fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) the OSC repository, embeded the new information into the existing catalog and merge. The steps below describe the process.\n", - "\n", - "0. (if needed) [Install git](https://github.com/git-guides/install-git) & create a [GitHub account](https://docs.github.com/en/get-started/start-your-journey/creating-an-account-on-github) \n", - "1. [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) the open science catalog repository on github - [https://github.com/ESA-EarthCODE/open-science-catalog-metadata](https://github.com/ESA-EarthCODE/open-science-catalog-metadata)\n", - "2. [Clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) your *forked* repository
\n", - " `git clone https://github.com/your-gh-username/open-science-catalog-metadata.git`\n", - "3. Set the current workspace to your **local clone** of the open science catalog metadata repository.
\n", - " `cd ./open-science-catalog-metadata/`\n", - "5. Create a new branch in the local clone
\n", - " `git checkout -b project_branch`\n", - "\n", - "\n", - "## 2. Embedding your newly created entries to the local copy of the open-science-catalog-metadata repository\n", - "\n", - "All OSC entries are interlinked to enable efficient search and analysis. For example, projects have associated products, themes, missions and in turn products link back to their projects, etc. Most of these can be automatically generated using the existing information in an OSC Entry and the associated `earthcode` library function.\n", - "\n", - "To use these functions you need a **local copy of the OSC**, preferably a fork, so that later, you can easily open a PR. The functions will save your newly created OSC entries and make changes to existing OSC entries, in order to conform to the required structure.\n", - "\n", - "### Import Python libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18e6e1d1-8f73-4ad5-92b0-fc6f35beba28", - "metadata": {}, - "outputs": [], - "source": [ - "# Import all necessary Python libraries to run the code\n", - "from pathlib import Path\n", - "from earthcode.git_add import (\n", - " save_product_collection_to_catalog,\n", - " save_project_collection_to_osc,\n", - " save_workflow_record_to_osc,\n", - " save_experiment_record_to_osc\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "e92d8d61-7274-49d5-bca0-0dc6873066ac", - "metadata": {}, - "source": [ - "### Specify the local path to your forked repository" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1eed3b9c", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# Specify the absolute path to the local OSC fork\n", - "catalog_root = Path('C:/Users/ewelina.dobrowolska/Documents/open-science-catalog-metadata/')\n", - "\n", - "# save the project entry and add the required links\n", - "save_project_collection_to_osc(project_collection, catalog_root)\n", - "\n", - "# save the product and add the required links\n", - "save_product_collection_to_catalog(product_collection, catalog_root)\n", - "\n", - "# save the workflow and add the required links\n", - "save_workflow_record_to_osc(workflow_record, catalog_root)\n", - "\n", - "# save the experiment and add the required links\n", - "save_experiment_record_to_osc(experiment_record, catalog_root)" - ] - }, - { - "cell_type": "markdown", - "id": "b135e726", - "metadata": {}, - "source": [ - "### 3. Validation\n", - "\n", - "There will be two types of checks before accepting your entry into the main OSC:\n", - "1. Automatic verification\n", - "2. Semantic validation\n", - "\n", - "\n", - "Before doing any of the checks you need to store your entries on disk in the OSC directory. This is required in order to check that all links are generated correctly. You can see the results of the automatic checks and any potential error using the library." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "e951a322", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# you can validate individual entries\n", - "from earthcode.validator import validateOSCEntry\n", - "validateOSCEntry(project_collection.to_dict(), catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "81fbb401-cdd1-4320-be9a-15ca555e7ce9", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "validateOSCEntry(product_collection.to_dict(), catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c3f5d2d-e272-4eec-bf00-b045c354dc60", - "metadata": {}, - "outputs": [], - "source": [ - "validateOSCEntry(workflow_record, catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d168ec36-7c1c-4e75-98a1-61b2d9197adc", - "metadata": {}, - "outputs": [], - "source": [ - "validateOSCEntry(experiment_record, catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "2357538c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([], [])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# you can also validate the whole catalog - YOU ARE NOT REQUESTED TO RUN THIS STEP AS IT MAY TAKE SOME TIME!\n", - "#from earthcode.validator import validate_catalog\n", - "#validate_catalog(catalog_root)" - ] - }, - { - "cell_type": "markdown", - "id": "1acdeec9", - "metadata": {}, - "source": [ - "## 4. Open a PR to add new entries\n", - "\n", - "After the validation passes, you are ready to request changes into existing [open-science-catalog-metadata](https://github.com/ESA-EarthCODE/open-science-catalog-metadata) repository to be able to publish your datasets and project. \n", - "By using the terminal: \n", - "1. Commit the changes to the newly created branch on your local copy of repository:
\n", - " `cd ./open-science-catalog-metadata`
\n", - " `git checkout -b \\`
\n", - " `git commit -m\"Adding new product\\_v2.0\"`
\n", - "3. [Push](https://docs.github.com/en/get-started/using-git/pushing-commits-to-a-remote-repository) the changes to your fork:
\n", - " `git push --set-upstream origin \\`\n", - "5. Open a [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) against the main open science catalog repository
\n", - " `gh pr create -f`" - ] - }, - { - "cell_type": "markdown", - "id": "78627b42-a2e4-4d09-af0b-c6cd903c7630", - "metadata": {}, - "source": [ - "## 5. Check the status of your PR direclty in GitHub\n", - "\n", - "After creation of Pull Request you should see it on the list: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/pulls\n", - "\n", - "Check the status of your PR under: https://github.com/ESA-EarthCODE/open-science-catalog-metadata/actions\n", - "\n", - "> Changes to the OSC content will be reviewed by the OSC Data Steward team. In case of any changes needed to your inputs, you will be contacted by the team." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/guide/2.1.Product_files_PRR.ipynb b/guide/2.1.Product_files_PRR.ipynb deleted file mode 100644 index 4346385..0000000 --- a/guide/2.1.Product_files_PRR.ipynb +++ /dev/null @@ -1,151 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "59a28ecf", - "metadata": {}, - "source": [ - "# 2.1 Request permanent storage of data on ESA cloud storage\n", - "\n", - "### 1. Storage\n", - "EarthCODE provides long term storage of research outcomes from ESA-funded EO Projects and activities including datasets and associated documents + workflows.
\n", - "Research outcomes are organised in STAC Objects (Collections), accessible via the STAC API accessible via the [STAC API](https://github.com/radiantearth/stac-api-spec) and the Open Science Catalogue Browser
\n", - "Each Collection contains [STAC Items](https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md), with their related Assets stored within the repository. \n", - "\n", - "The upload process to the PRR is manual at the moment, therefore you will have to [email us](mailto:earth-code.esa.int) with information about your:\n", - "- Data type\n", - "- Data size\n", - "- ESA contract and ESA TO for your project\n", - "- STAC items describing the files\n", - "**A pre-requisite to upload the data is to have descriptions of each file using STAC items. For examples how to do this see section 3.**\n", - "\n", - "\n", - "## 2. Format\n", - "\n", - "We strongly encourage [cloud-optimised format for your data](https://esa-earthcode.github.io/documentation/Community%20and%20Best%20Practices/Data%20and%20Workflow%20Best%20Practices/Data/), since it makes storage and access much easier. If your data is already in one of the preferred formats - `cog`, `parquet`, `zarr`, etc - there is nothing to do for this step.\n", - "\n", - "If the data is not in a [cloud-optimised format](https://esa-earthcode.github.io/documentation/Community%20and%20Best%20Practices/Data%20and%20Workflow%20Best%20Practices/Data/), we encourage you to transform it yourselves or to contact us and we can help doing this. \n", - "\n", - "## 3. File-level metadata\n", - "\n", - "This is the most time-consuming step. There are multiple strategies for doing this, we are flexible and it is up to you to decide how to do it, so long as the data conforms to standard STAC specification.
**The main consideration should be usability of the data!**
You can learn more about STAC specification here: https://stacspec.org/en\n", - "\n", - "If you are new to STAC Specification and how this applies to your dataset, we have many [tutorials](https://esa-earthcode.github.io/tutorials/index-1/) available from the EarthCODE Portal and executable from a designated [workspace](https://workspace.earthcode.eox.at/). The tutorials examples how to generate the STAC Items from most commonly used data formats like: `netcdf, tiff and zarr files`. You can start with the introductory tutorial with will also have an overview of all the information provided here: https://esa-earthcode.github.io/tutorials/prr-stac-introduction/ . Note, that the code in the examples does not generalise fully, so we only offer a few libraries and pointers to get you started. You have to tailor the code to your data, but generally the list of tutorials should faciliate this task. You can run all examples in the earthcode library environment.\n", - "\n", - "More manual way to create STAC Items and Asset level data, is shown in the following [example](https://esa-earthcode.github.io/tutorials/creating-stac-catalog-from-prr-example/) (applicable to all file types - including documentation)
\n", - "\n", - "*The provided example use Python programming language, but you are free to explore options in other programming languages, if your are more comfortable with them. In that case please share with us the STAC Collections generated by your script*. \n", - "\n", - "> We can support you through this all stages of this process, just [**contact us**](mailto:earth-code.esa.int) or post in the [FORUM](https://discourse-earthcode.eox.at/c/technical-support/8)!\n", - "\n", - "\n", - "\n", - "##### 2.1.2.1 Adding PRR files\n", - "\n", - "- After you have sent us the data with the STAC items and we have returned a link for and hosted your data. You can continue the OSC process.\n", - "\n", - " You can run fill the below URL, with the one we returned to you and run the next section of the notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ef00373", - "metadata": {}, - "outputs": [], - "source": [ - "from earthcode.git_add import save_item_links_to_product_collection\n", - "from earthcode.validator import validate_catalog\n", - "from pathlib import Path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6fe1b477", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "\n", - "# Define the relevant data links to be manually added\n", - "# link to an external data collection\n", - "item_link = ''\n", - "# Link to accessing the data, this link is required.\n", - "access_link = f'https://opensciencedata.esa.int/stac-browser/#/external/{item_link}'\n", - "#Link to the documentation, leave as None, if not available\n", - "documentation_link = ''\n", - "\n", - "# the ID of the product you created in the previous steps\n", - "product_id = ''\n", - "\n", - "catalog_root = str(Path(\"../open-science-catalog-metadata\").resolve())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12472fa5", - "metadata": {}, - "outputs": [], - "source": [ - "catalog_root = Path(catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c060d917", - "metadata": {}, - "outputs": [], - "source": [ - "save_item_links_to_product_collection(catalog_root, product_id, item_link, access_link, documentation_link)" - ] - }, - { - "cell_type": "markdown", - "id": "8cead67f", - "metadata": {}, - "source": [ - "# Run validation to make sure everything works." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac0f412a", - "metadata": {}, - "outputs": [], - "source": [ - "## run validation\n", - "errors, error_files = validate_catalog(catalog_root)\n", - "if errors or error_files:\n", - " raise AssertionError(f\"Catalog validation failed. errors={len(errors)} files={len(error_files)}\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/guide/2.1.Product_files_self_hosted.ipynb b/guide/2.1.Product_files_self_hosted.ipynb deleted file mode 100644 index 6dec87f..0000000 --- a/guide/2.1.Product_files_self_hosted.ipynb +++ /dev/null @@ -1,171 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6f2fba0e", - "metadata": {}, - "source": [ - "# 2.2 Product files - self-hosted\n", - "\n", - "To add your data to the Open Science Catalog, you have to generate a STAC items that describes your files and associated documentation.\n", - "\n", - "> We have many [tutorials](https://esa-earthcode.github.io/tutorials/index-1/) available from the EarthCODE Portal and executable from a designated [workspace](https://workspace.earthcode.eox.at/). The tutorials examples how to generate the STAC Items from most commonly used data formats like: `netcdf, tiff and zarr files`. **Note, that the code in the examples/tutorials and below does not generalise fully. You will have to tailor the code to your data.** \n", - "\n", - "# 2.1.1 Generating STAC items\n", - "\n", - "Below you can see an example of a more manual approach to creating STAC items and adding them to an existing OSC product (created and saved in the previous stage.) You can fill the information for a single file item and create the STAC item this way for each file. **If you have multiple files, you will have to update the code accordingly.**\n", - "\n", - "> We can support you through this all stages of this process, just [**contact us**](mailto:earth-code.esa.int) or post in the [FORUM](https://discourse-earthcode.eox.at/c/technical-support/8)!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b6f59b3", - "metadata": {}, - "outputs": [], - "source": [ - "from earthcode.git_add import save_item_to_product_collection\n", - "from earthcode.validator import validate_catalog\n", - "from pathlib import Path\n", - "from earthcode.metadata_input_definitions import ItemMetadata\n", - "from earthcode.static import create_item\n", - "import json\n", - "import shapely\n", - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "535d2e4d", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "catalog_root = str(Path(\"../open-science-catalog-metadata\").resolve())\n", - "\n", - "product_id = \"\"\n", - "\n", - "# Define the relevant links to complete your data description.\n", - "access_link = \"\"\n", - "documentation_link = \"\"\n", - "license_link = \"\"\n", - "item_link = \"\"\n", - "item_title = \"\"\n", - "\n", - "# Item metadata (for adding one item to the product collection)\n", - "item_id = \"\"\n", - "item_bbox = [-180.0, -90.0, 180.0, 90.0]\n", - "item_datetime = \"\"\n", - "item_license = \"\"\n", - "item_description = \"\"\n", - "item_data_url = \"\"\n", - "item_data_mime_type = \"\"\n", - "item_data_title = \"\"\n", - "item_extra_fields = {}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4af0614", - "metadata": {}, - "outputs": [], - "source": [ - "catalog_root = Path(catalog_root)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "224e316a", - "metadata": {}, - "outputs": [], - "source": [ - "## generate a STAC item associated with a specific collection\n", - "geometry = json.loads(json.dumps(shapely.box(*item_bbox).__geo_interface__))\n", - "item_metadata = ItemMetadata(\n", - " itemid=item_id,\n", - " geometry=geometry,\n", - " data_time=pd.to_datetime(item_datetime),\n", - " bbox=item_bbox,\n", - " product_id=product_id,\n", - " license=item_license,\n", - " description=item_description,\n", - " data_url=item_data_url,\n", - " data_mime_type=item_data_mime_type,\n", - " data_title=item_data_title,\n", - " extra_fields=item_extra_fields,\n", - ")\n", - "item = create_item(item_metadata)" - ] - }, - { - "cell_type": "markdown", - "id": "da773887", - "metadata": {}, - "source": [ - "##### 2.1.2 Adding STAC items to existing OSC product collections\n", - "\n", - "If the files are remote, and you've generated the items use the below function to read all items and connect them to the product collection." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f5762ba", - "metadata": {}, - "outputs": [], - "source": [ - "# save item to collection and add backlinks\n", - "save_item_to_product_collection(item, product_id, catalog_root)" - ] - }, - { - "cell_type": "markdown", - "id": "8cead67f", - "metadata": {}, - "source": [ - "# Run validation to make sure everything works." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "473cb389", - "metadata": {}, - "outputs": [], - "source": [ - "## run validation\n", - "errors, error_files = validate_catalog(catalog_root)\n", - "if errors or error_files:\n", - " raise AssertionError(f\"Catalog validation failed. errors={len(errors)} files={len(error_files)}\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index fe598e8..680c75b 100644 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -22,7 +22,7 @@ def catalog_root(tmp_path: Path) -> Path: def test_notebooks(catalog_root:Path): repo_root = Path(__file__).resolve().parents[1] - notebooks_dir = repo_root / "guide" + notebooks_dir = repo_root / "docs/guides/" unique = "123456" project_id = f"4datlantic-ohc{unique}" project_title = "4DAtlantic-OHC" @@ -103,10 +103,8 @@ def test_notebooks(catalog_root:Path): "project_id": project_id, "project_title": project_title, "workflow_themes": ["oceans"], - "workflow_contracts_info": [("Magellium", "contact@magellium.fr")], - "codeurl": "https://github.com/ESA-EarthCODE/open-science-catalog-metadata", + "workflow_url": "https://github.com/ESA-EarthCODE/open-science-catalog-metadata", "workflow_doi": None, - "include_workflow_bbox": False, "include_workflow_time": False, } @@ -120,47 +118,14 @@ def test_notebooks(catalog_root:Path): "experiment_formats": ["GeoTIFF"], "experiment_themes": ["oceans"], "experiment_input_parameters_link": "https://github.com/deepesdl/cube-gen", - "experiment_enviroment_link": "https://github.com/deepesdl/cube-gen", + "experiment_environment_link": "https://github.com/deepesdl/cube-gen", "workflow_id": workflow_id, "workflow_title": workflow_title, "product_id": product_id, "product_title": product_title, "experiment_contacts": None, - "include_experiment_bbox": False, "include_experiment_time": False, - } - - prr_collection_params = { - "item_link" : "https://s3.waw4-1.cloudferro.com/EarthCODE/Catalogs/4datlantic-ohc/collection.json", - "documentation_link": "https://www.aviso.altimetry.fr/fileadmin/documents/data/tools/OHC-EEI/OHCATL-DT-035-MAG_EDD_V3.0.pdf", - "license_link": "https://www.aviso.altimetry.fr/fileadmin/documents/data/License_Aviso.pdf", - "access_link": "https://s3.waw4-1.cloudferro.com/EarthCODE/Catalogs/4datlantic-ohc/collection.json", - "product_id": product_id, - "catalog_root": str(catalog_root), - } - - remote_item_params = { - "item_title": product_title, - "item_id": f"{product_id}-zip_folder", - "item_bbox": [-180.0, -90.0, 180.0, 90.0], - "item_datetime": "2024-07-16T00:00:00Z", - "item_license": "CC-BY-4.0", - "item_description": ( - "Dataset contents for notebook-run validation item." - ), - "item_data_url": "https://wgms.ch/downloads/GlaMBIE_Data_DOI_10.5904_wgms-glambie-2024-07.zip", - "item_data_mime_type": "application/zip", - "item_data_title": "GlaMBIE dataset archive (ZIP)", - "item_extra_fields": { - "file:size": 2726298, - "file:compression": "zip", - "data:format": "CSV inside ZIP", - }, - - "product_id": product_id, - "catalog_root": str(catalog_root), - } - + } # execute notebooks, since each runs a validation on the entire catalog # we are also checking for newly introduced errors. @@ -186,26 +151,6 @@ def test_notebooks(catalog_root:Path): log_output=True, ) - # add PRR product items - pm.execute_notebook( - input_path=str(notebooks_dir / "2.1.Product_files_PRR.ipynb"), - output_path=None, - parameters=prr_collection_params, - kernel_name="python3", - cwd=str(repo_root), - log_output=True, - ) - - # add remote product items - pm.execute_notebook( - input_path=str(notebooks_dir / "2.1.Product_files_self_hosted.ipynb"), - output_path=None, - parameters=remote_item_params, - kernel_name="python3", - cwd=str(repo_root), - log_output=True, - ) - # workflow pm.execute_notebook( input_path=str(notebooks_dir / "3.Workflow.ipynb"),