Skip to content

Commit 85942fc

Browse files
jonhealy1jonhealy1
authored
Merge pull request #5 from jonhealy1/data_loader
Data loader
2 parents 30383bf + fa3e4b6 commit 85942fc

File tree

7 files changed

+161
-4
lines changed

7 files changed

+161
-4
lines changed

‎Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,8 @@ pybase-install:
4040

4141
.PHONY: install
4242
install: pybase-install
43-
pip install -e ./stac_fastapi/mongo[dev,server]
43+
pip install -e ./stac_fastapi/mongo[dev,server]
44+
45+
.PHONY: ingest
46+
ingest:
47+
python3 data_loader/data_loader.py

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.3.1
1+
2.3.2

‎data_loader/data_loader.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
"""Database ingestion script."""
2+
import json
3+
import os
4+
5+
import click
6+
import requests
7+
8+
DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/")
9+
STAC_API_BASE_URL = "http://localhost:8083"
10+
11+
12+
def load_data(filename):
13+
"""Load json data."""
14+
with open(os.path.join(DATA_DIR, filename)) as file:
15+
return json.load(file)
16+
17+
18+
def load_collection(collection_id):
19+
"""Load stac collection into the database."""
20+
collection = load_data("collection.json")
21+
collection["id"] = collection_id
22+
try:
23+
resp = requests.post(f"{STAC_API_BASE_URL}/collections", json=collection)
24+
if resp.status_code == 200:
25+
print(f"Status code: {resp.status_code}")
26+
print(f"Added collection: {collection['id']}")
27+
elif resp.status_code == 409:
28+
print(f"Status code: {resp.status_code}")
29+
print(f"Collection: {collection['id']} already exists")
30+
except requests.ConnectionError:
31+
click.secho("failed to connect")
32+
33+
34+
def load_items():
35+
"""Load stac items into the database."""
36+
feature_collection = load_data("sentinel-s2-l2a-cogs_0_100.json")
37+
collection = "test-collection"
38+
load_collection(collection)
39+
40+
for feature in feature_collection["features"]:
41+
try:
42+
feature["stac_extensions"] = []
43+
feature["stac_version"] = "1.0.0"
44+
feature["collection"] = collection
45+
resp = requests.post(
46+
f"{STAC_API_BASE_URL}/collections/{collection}/items", json=feature
47+
)
48+
49+
if resp.status_code == 200:
50+
print(f"Status code: {resp.status_code}")
51+
print(f"Added item: {feature['id']}")
52+
elif resp.status_code == 409:
53+
print(f"Status code: {resp.status_code}")
54+
print(f"Item: {feature['id']} already exists")
55+
except requests.ConnectionError:
56+
click.secho("failed to connect")
57+
58+
59+
load_items()

‎data_loader/setup_data/collection.json‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
{
2+
"id":"sentinel-s2-l2a-cogs",
3+
"stac_version":"1.0.0-beta.2",
4+
"description":"Sentinel-2a and Sentinel-2b imagery, processed to Level 2A (Surface Reflectance) and converted to Cloud-Optimized GeoTIFFs",
5+
"links":[
6+
{"rel":"self","href":"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs"},
7+
{"rel":"license","href":"https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice"},
8+
{"rel":"about","href":"https://github.com/stac-utils/stac-sentinel"},
9+
{"rel":"parent","href":"https://earth-search.aws.element84.com/v0/"},
10+
{"rel":"root","href":"https://earth-search.aws.element84.com/v0/"},
11+
{"rel":"items","href":"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items"}
12+
],
13+
"stac_extensions":["item-assets"],
14+
"title":"Sentinel 2 L2A COGs",
15+
"keywords":["sentinel","earth observation","esa"],
16+
"providers":[
17+
{"name":"ESA","roles":["producer"],"url":"https://earth.esa.int/web/guest/home"},
18+
{"name":"Sinergise","roles":["processor"],"url":"https://registry.opendata.aws/sentinel-2/"},
19+
{"name":"AWS","roles":["host"],"url":"http://sentinel-pds.s3-website.eu-central-1.amazonaws.com/"},
20+
{"name":"Element 84","roles":["processor"],"url":"https://element84.com"}
21+
],
22+
"summaries":{
23+
"platform":["sentinel-2a","sentinel-2b"],
24+
"constellation":["sentinel-2"],
25+
"instruments":["msi"],
26+
"gsd":[10],
27+
"view:off_nadir":[0]
28+
},
29+
"item_assets":{
30+
"thumbnail":{
31+
"title":"Thumbnail",
32+
"type":"image/png",
33+
"roles":["thumbnail"]
34+
},
35+
"overview":{
36+
"title":"True color image",
37+
"type":"image/tiff; application=geotiff; profile=cloud-optimized",
38+
"roles":["overview"],
39+
"gsd":10,
40+
"eo:bands":[
41+
{"name":"B04","common_name":"red","center_wavelength":0.6645,"full_width_half_max":0.038},
42+
{"name":"B03","common_name":"green","center_wavelength":0.56,"full_width_half_max":0.045},
43+
{"name":"B02","common_name":"blue","center_wavelength":0.4966,"full_width_half_max":0.098}
44+
]
45+
},
46+
"info":{
47+
"title":"Original JSON metadata",
48+
"type":"application/json",
49+
"roles":["metadata"]
50+
},
51+
"metadata":{
52+
"title":"Original XML metadata",
53+
"type":"application/xml",
54+
"roles":["metadata"]
55+
},
56+
"visual":{
57+
"title":"True color image",
58+
"type":"image/tiff; application=geotiff; profile=cloud-optimized",
59+
"roles":["overview"],
60+
"gsd":10,
61+
"eo:bands":[
62+
{"name":"B04","common_name":"red","center_wavelength":0.6645,"full_width_half_max":0.038},
63+
{"name":"B03","common_name":"green","center_wavelength":0.56,"full_width_half_max":0.045},
64+
{"name":"B02","common_name":"blue","center_wavelength":0.4966,"full_width_half_max":0.098}
65+
]
66+
},
67+
"B01":{
68+
"title":"Band 1 (coastal)",
69+
"type":"image/tiff; application=geotiff; profile=cloud-optimized",
70+
"roles":["data"],
71+
"gsd":60,
72+
"eo:bands":[
73+
{"name":"B01","common_name":"coastal","center_wavelength":0.4439,"full_width_half_max":0.027}
74+
]
75+
},
76+
"B02":{
77+
"title":"Band 2 (blue)",
78+
"type":"image/tiff; application=geotiff; profile=cloud-optimized",
79+
"roles":["data"],
80+
"gsd":10,
81+
"eo:bands":[
82+
{"name":"B02","common_name":"blue","center_wavelength":0.4966,"full_width_half_max":0.098}
83+
]
84+
},
85+
"B03":{
86+
"title":"Band 3 (green)",
87+
"type":"image/tiff; application=geotiff; profile=cloud-optimized",
88+
"roles":["data"],
89+
"gsd":10,
90+
"eo:bands":[
91+
{"name":"B03","common_name":"green","center_wavelength":0.56,"full_width_half_max":0.045}
92+
]
93+
},"B04":{"title":"Band 4 (red)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":10,"eo:bands":[{"name":"B04","common_name":"red","center_wavelength":0.6645,"full_width_half_max":0.038}]},"B05":{"title":"Band 5","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B05","center_wavelength":0.7039,"full_width_half_max":0.019}]},"B06":{"title":"Band 6","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B06","center_wavelength":0.7402,"full_width_half_max":0.018}]},"B07":{"title":"Band 7","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B07","center_wavelength":0.7825,"full_width_half_max":0.028}]},"B08":{"title":"Band 8 (nir)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":10,"eo:bands":[{"name":"B08","common_name":"nir","center_wavelength":0.8351,"full_width_half_max":0.145}]},"B8A":{"title":"Band 8A","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B8A","center_wavelength":0.8648,"full_width_half_max":0.033}]},"B09":{"title":"Band 9","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":60,"eo:bands":[{"name":"B09","center_wavelength":0.945,"full_width_half_max":0.026}]},"B11":{"title":"Band 11 (swir16)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B11","common_name":"swir16","center_wavelength":1.6137,"full_width_half_max":0.143}]},"B12":{"title":"Band 12 (swir22)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"gsd":20,"eo:bands":[{"name":"B12","common_name":"swir22","center_wavelength":2.22024,"full_width_half_max":0.242}]},"AOT":{"title":"Aerosol Optical Thickness (AOT)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"]},"WVP":{"title":"Water Vapour (WVP)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"]},"SCL":{"title":"Scene Classification Map (SCL)","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"]}},"extent":{"spatial":{"bbox":[[-180,-90,180,90]]},"temporal":{"interval":[["2015-06-27T10:25:31.456000Z",null]]}},"license":"proprietary"}

‎data_loader/setup_data/sentinel-s2-l2a-cogs_0_100.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎stac_fastapi/mongo/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
keywords="STAC FastAPI COG",
4848
author=u"Arturo Engineering",
4949
author_email="engineering@arturo.ai",
50-
url="https://github.com/jonhealy1/stac-fastapi-nosql",
50+
url="https://github.com/jonhealy1/stac-fastapi-mongo",
5151
license="MIT",
5252
packages=find_namespace_packages(exclude=["alembic", "tests", "scripts"]),
5353
zip_safe=False,

‎stac_fastapi/mongo/stac_fastapi/mongo/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
"""library version."""
2-
__version__ = "2.3.1"
2+
__version__ = "2.3.2"

0 commit comments

Comments
 (0)