Update Sentinel-5P

datasets/io-land-cover/dataset.yaml

@@ -2,7 +2,7 @@ id: io_lulc
 image: ${{ args.registry }}/pctasks-task-base:latest
 
 args:
-- registry
+  - registry
 
 code:
   src: ${{ local.path(./io_lulc.py) }}

datasets/ms-buildings/Dockerfile

@@ -0,0 +1,74 @@
+FROM ubuntu:20.04
+
+# Setup timezone info
+ENV TZ=UTC
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update && apt-get install -y software-properties-common
+
+RUN add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    apt-get install -y build-essential python3-dev python3-pip \
+    jq unzip ca-certificates wget curl git && \
+    apt-get autoremove && apt-get autoclean && apt-get clean
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+# See https://github.com/mapbox/rasterio/issues/1289
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+# Install Python 3.11
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
+    && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \
+    && rm -rf "Mambaforge-$(uname)-$(uname -m).sh"
+
+ENV PATH /opt/conda/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
+
+RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy
+
+RUN python -m pip install --upgrade pip
+
+# Install common packages
+COPY requirements-task-base.txt /tmp/requirements.txt
+RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
+
+#
+# Copy and install packages
+#
+
+COPY pctasks/core /opt/src/pctasks/core
+RUN cd /opt/src/pctasks/core && \
+    pip install .
+
+COPY pctasks/cli /opt/src/pctasks/cli
+RUN cd /opt/src/pctasks/cli && \
+    pip install .
+
+COPY pctasks/task /opt/src/pctasks/task
+RUN cd /opt/src/pctasks/task && \
+    pip install .
+
+COPY pctasks/client /opt/src/pctasks/client
+RUN cd /opt/src/pctasks/client && \
+    pip install .
+
+COPY pctasks/ingest /opt/src/pctasks/ingest
+RUN cd /opt/src/pctasks/ingest && \
+    pip install .
+
+COPY pctasks/dataset /opt/src/pctasks/dataset
+RUN cd /opt/src/pctasks/dataset && \
+    pip install .
+
+COPY ./datasets/ms-buildings/requirements.txt /opt/src/datasets/ms-buildings/requirements.txt
+RUN python3 -m pip install -r /opt/src/datasets/ms-buildings/requirements.txt
+
+# Setup Python Path to allow import of test modules
+ENV PYTHONPATH=/opt/src:$PYTHONPATH
+
+WORKDIR /opt/src

datasets/sentinel-5p/Dockerfile

@@ -0,0 +1,74 @@
+FROM ubuntu:20.04
+
+# Setup timezone info
+ENV TZ=UTC
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update && apt-get install -y software-properties-common
+
+RUN add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    apt-get install -y build-essential python3-dev python3-pip \
+    jq unzip ca-certificates wget curl git && \
+    apt-get autoremove && apt-get autoclean && apt-get clean
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+# See https://github.com/mapbox/rasterio/issues/1289
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+# Install Python 3.8
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
+    && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \
+    && rm -rf "Mambaforge-$(uname)-$(uname -m).sh"
+
+ENV PATH /opt/conda/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
+
+RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5
+
+RUN python -m pip install --upgrade pip
+
+# Install common packages
+COPY requirements-task-base.txt /tmp/requirements.txt
+RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
+
+#
+# Copy and install packages
+#
+
+COPY pctasks/core /opt/src/pctasks/core
+RUN cd /opt/src/pctasks/core && \
+    pip install .
+
+COPY pctasks/cli /opt/src/pctasks/cli
+RUN cd /opt/src/pctasks/cli && \
+    pip install .
+
+COPY pctasks/task /opt/src/pctasks/task
+RUN cd /opt/src/pctasks/task && \
+    pip install .
+
+COPY pctasks/client /opt/src/pctasks/client
+RUN cd /opt/src/pctasks/client && \
+    pip install .
+
+COPY pctasks/ingest /opt/src/pctasks/ingest
+RUN cd /opt/src/pctasks/ingest && \
+    pip install .
+
+COPY pctasks/dataset /opt/src/pctasks/dataset
+RUN cd /opt/src/pctasks/dataset && \
+    pip install .
+
+COPY ./datasets/sentinel-5p/requirements.txt /opt/src/datasets/sentinel-5p/requirements.txt
+RUN python3 -m pip install -r /opt/src/datasets/sentinel-5p/requirements.txt
+
+# Setup Python Path to allow import of test modules
+ENV PYTHONPATH=/opt/src:$PYTHONPATH
+
+WORKDIR /opt/src

datasets/sentinel-5p/README.md

@@ -1 +1,11 @@
-# Work in progress - in a broken state
+# planetary-computer-tasks dataset: sentinel-5p
+
+Sentinel 5 Precursor
+
+## Building the Docker image
+
+To build and push a custom docker image to our container registry:
+
+```shell
+az acr build -r {the registry} --subscription {the subscription} -t pctasks-sentinel-5p:latest -f datasets/sentinel-5p/Dockerfile .
+```

datasets/sentinel-5p/collection/description.md

@@ -0,0 +1,19 @@
+The Copernicus [Sentinel-5 Precursor](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-5p) mission provides high spatio-temporal resolution measurements of the Earth's atmosphere. The mission consists of one satellite carrying the [TROPOspheric Monitoring Instrument](http://www.tropomi.eu/) (TROPOMI). The satellite flies in loose formation with NASA's [Suomi NPP](https://www.nasa.gov/mission_pages/NPP/main/index.html) spacecraft, allowing utilization of co-located cloud mask data provided by the [Visible Infrared Imaging Radiometer Suite](https://www.nesdis.noaa.gov/current-satellite-missions/currently-flying/joint-polar-satellite-system/visible-infrared-imaging) (VIIRS) instrument onboard Suomi NPP during processing of the TROPOMI methane product.
+
+The Sentinel-5 Precursor mission aims to reduce the global atmospheric data gap between the retired [ENVISAT](https://earth.esa.int/eogateway/missions/envisat) and [AURA](https://www.nasa.gov/mission_pages/aura/main/index.html) missions and the future [Sentinel-5](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-5) mission. Sentinel-5 Precursor [Level 2 data](http://www.tropomi.eu/data-products/level-2-products) provide total columns of ozone, sulfur dioxide, nitrogen dioxide, carbon monoxide and formaldehyde, tropospheric columns of ozone, vertical profiles of ozone and cloud & aerosol information. These measurements are used for improving air quality forecasts and monitoring the concentrations of atmospheric constituents.
+
+This STAC Collection provides Sentinel-5 Precursor Level 2 data, in NetCDF format, since April 2018 for the following products:
+
+* [`L2__AER_AI`](http://www.tropomi.eu/data-products/uv-aerosol-index): Ultraviolet aerosol index
+* [`L2__AER_LH`](http://www.tropomi.eu/data-products/aerosol-layer-height): Aerosol layer height
+* [`L2__CH4___`](http://www.tropomi.eu/data-products/methane): Methane (CH<sub>4</sub>) total column
+* [`L2__CLOUD_`](http://www.tropomi.eu/data-products/cloud): Cloud fraction, albedo, and top pressure
+* [`L2__CO____`](http://www.tropomi.eu/data-products/carbon-monoxide): Carbon monoxide (CO) total column
+* [`L2__HCHO__`](http://www.tropomi.eu/data-products/formaldehyde): Formaldehyde (HCHO) total column
+* [`L2__NO2___`](http://www.tropomi.eu/data-products/nitrogen-dioxide): Nitrogen dioxide (NO<sub>2</sub>) total column
+* [`L2__O3____`](http://www.tropomi.eu/data-products/total-ozone-column): Ozone (O<sub>3</sub>) total column
+* [`L2__O3_TCL`](http://www.tropomi.eu/data-products/tropospheric-ozone-column): Ozone (O<sub>3</sub>) tropospheric column
+* [`L2__SO2___`](http://www.tropomi.eu/data-products/sulphur-dioxide): Sulfur dioxide (SO<sub>2</sub>) total column
+* [`L2__NP_BD3`](http://www.tropomi.eu/data-products/auxiliary): Cloud from the Suomi NPP mission, band 3
+* [`L2__NP_BD6`](http://www.tropomi.eu/data-products/auxiliary): Cloud from the Suomi NPP mission, band 6
+* [`L2__NP_BD7`](http://www.tropomi.eu/data-products/auxiliary): Cloud from the Suomi NPP mission, band 7