Skip to content

Commit ee4e499

Browse files
danovarodanovaro
committed
Merge branch 'develop' of https://github.com/ecmwf/fdb into develop
2 parents 3c0eab4 + a8c9c64 commit ee4e499

File tree

144 files changed

+6765
-58
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

144 files changed

+6765
-58
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ jobs:
2929
fdb: ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}
3030
codecov_upload: true
3131
clang_format: true
32+
clang_format_ignore: |
33+
third_party
3234
secrets: inherit
3335

3436
# Run CI of private downstream packages on self-hosted runners

‎.github/workflows/docs.yml‎

Lines changed: 78 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
tags:
1010
- '**'
1111
jobs:
12-
build:
12+
build-fdb:
1313
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' || github.event_name == 'push'}}
1414
runs-on: ubuntu-latest
1515
outputs:
@@ -23,33 +23,34 @@ jobs:
2323
python-version: '3.13'
2424
- name: Install python dependencies
2525
run: |
26-
python -m pip install -r docs/requirements.txt
26+
python -m pip install -r docs/fdb/requirements.txt
2727
- name: Prepare Doxygen
2828
uses: ecmwf/reusable-workflows/setup-doxygen@main
2929
with:
3030
version: 1.14.0
3131
- name: Build doc
3232
run: |
33-
DOCBUILD_OUTPUT=docs_out ./docs/build_docs.sh
33+
DOCBUILD_OUTPUT=docs_out ./docs/fdb/build_docs.sh
3434
- name: Archive documentation
3535
id: upload-doc
3636
uses: actions/upload-artifact@v4
3737
with:
38-
name: documentation
38+
name: documentation-fdb
3939
path: docs_out/sphinx
40-
preview-publish:
40+
preview-publish-fdb:
4141
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }}
42-
needs: build
42+
needs: build-fdb
4343
uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-publish.yml@main
4444
with:
45-
artifact-id: ${{ needs.build.outputs.artifact-id }}
45+
artifact-id: ${{ needs.build-fdb.outputs.artifact-id }}
4646
space: docs
4747
name: dev-section
4848
path: fdb/pull-requests
49-
link-text: 🌈🌦️📖🚧 Documentation 🚧📖🌦️🌈
49+
link-tag: PREVIEW-PUBLISH-FDB
50+
link-text: 🌈🌦️📖🚧 Documentation FDB 🚧📖🌦️🌈
5051
secrets:
5152
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}
52-
preview-unpublish:
53+
preview-unpublish-fdb:
5354
if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
5455
uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-unpublish.yml@main
5556
with:
@@ -58,17 +59,17 @@ jobs:
5859
path: fdb/pull-requests
5960
secrets:
6061
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}
61-
publish:
62+
publish-fdb:
6263
if: >-
6364
${{ github.event_name == 'push' && (
6465
github.ref_name == 'develop' ||
6566
github.ref_name == 'master' ||
6667
github.ref_type == 'tag'
6768
) }}
68-
needs: build
69+
needs: build-fdb
6970
uses: ecmwf/reusable-workflows/.github/workflows/docs-publish.yml@main
7071
with:
71-
artifact-id: ${{ needs.build.outputs.artifact-id }}
72+
artifact-id: ${{ needs.build-fdb.outputs.artifact-id }}
7273
space: docs
7374
name: dev-section
7475
path: fdb
@@ -77,3 +78,68 @@ jobs:
7778
|| github.ref_name == 'develop' && 'latest' }}
7879
secrets:
7980
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}
81+
build-z3fdb:
82+
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' || github.event_name == 'push'}}
83+
runs-on: ubuntu-latest
84+
outputs:
85+
artifact-id: ${{ steps.upload-doc.outputs.artifact-id }}
86+
steps:
87+
- name: Checkout
88+
uses: actions/checkout@v4
89+
- name: Setup Python
90+
uses: actions/setup-python@v5
91+
with:
92+
python-version: '3.13'
93+
- name: Install python dependencies
94+
run: |
95+
python -m pip install -r docs/z3fdb/requirements.txt
96+
- name: Build doc
97+
run: |
98+
DOCBUILD_OUTPUT=docs_out ./docs/z3fdb/build_docs.sh
99+
- name: Archive documentation
100+
id: upload-doc
101+
uses: actions/upload-artifact@v4
102+
with:
103+
name: documentation-z3fdb
104+
path: docs_out/sphinx
105+
preview-publish-z3fdb:
106+
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }}
107+
needs: build-z3fdb
108+
uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-publish.yml@main
109+
with:
110+
artifact-id: ${{ needs.build-z3fdb.outputs.artifact-id }}
111+
space: docs
112+
name: dev-section
113+
path: z3fdb/pull-requests
114+
link-tag: PREVIEW-PUBLISH-Z3FDB
115+
link-text: 🌈🌦️📖🚧 Documentation Z3FDB 🚧📖🌦️🌈
116+
secrets:
117+
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}
118+
preview-unpublish-z3fdb:
119+
if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
120+
uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-unpublish.yml@main
121+
with:
122+
space: docs
123+
name: dev-section
124+
path: z3fdb/pull-requests
125+
secrets:
126+
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}
127+
publish:
128+
if: >-
129+
${{ github.event_name == 'push' && (
130+
github.ref_name == 'develop' ||
131+
github.ref_name == 'master' ||
132+
github.ref_type == 'tag'
133+
) }}
134+
needs: build-z3fdb
135+
uses: ecmwf/reusable-workflows/.github/workflows/docs-publish.yml@main
136+
with:
137+
artifact-id: ${{ needs.build-z3fdb.outputs.artifact-id }}
138+
space: docs
139+
name: dev-section
140+
path: z3fdb
141+
id: ${{ github.ref_name }}
142+
softlink: ${{ github.ref_name == 'master' && 'stable'
143+
|| github.ref_name == 'develop' && 'latest' }}
144+
secrets:
145+
sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }}

‎.github/workflows/z3fdb.yml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
name: Build and Test z3fdb
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch: # Allows manual trigger
6+
7+
jobs:
8+
prepare-deps:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Get ecbuild
12+
uses: actions/checkout@v5
13+
with:
14+
repository: ecmwf/ecbuild
15+
ref: develop
16+
path: ecbuild
17+
- name: Get cxx-dependencies
18+
uses: actions/checkout@v5
19+
with:
20+
repository: ecmwf/cxx-dependencies
21+
ref: master
22+
path: cxx-dependencies-src
23+
token: ${{ secrets.GH_REPO_READ_TOKEN }}
24+
submodules: recursive
25+
- name: Install dependencies
26+
run: |
27+
mkdir cxx-dependencies-build
28+
BUILD_PATH=cxx-dependencies-build INSTALL_PREFIX=dependencies cxx-dependencies-src/build.sh
29+
- name: Get eccodes
30+
uses: actions/checkout@v5
31+
with:
32+
repository: ecmwf/eccodes
33+
ref: develop
34+
path: eccodes-src
35+
- name: Install eccodes
36+
run: |
37+
mkdir eccodes-build
38+
cmake \
39+
-B eccodes-build \
40+
-S eccodes-src \
41+
-GNinja \
42+
-DCMAKE_INSTALL_PREFIX=dependencies \
43+
-DCMAKE_PREFIX_PATH=dependencies \
44+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
45+
-DENABLE_MEMFS=ON \
46+
-DENABLE_AEC=ON
47+
cmake --build eccodes-build -j -t install
48+
- name: Get eckit
49+
uses: actions/checkout@v5
50+
with:
51+
repository: ecmwf/eckit
52+
ref: develop
53+
path: eckit-src
54+
- name: Install eckit
55+
run: |
56+
mkdir eckit-build
57+
cmake \
58+
-B eckit-build \
59+
-S eckit-src \
60+
-GNinja \
61+
-DCMAKE_INSTALL_PREFIX=dependencies \
62+
-DCMAKE_PREFIX_PATH=dependencies \
63+
-DCMAKE_BUILD_TYPE=RelWithDebInfo
64+
cmake --build eckit-build -j -t install
65+
- name: Get metkit
66+
uses: actions/checkout@v5
67+
with:
68+
repository: ecmwf/metkit
69+
ref: develop
70+
path: metkit-src
71+
- name: Install metkit
72+
run: |
73+
mkdir metkit-build
74+
cmake \
75+
-B metkit-build \
76+
-S metkit-src \
77+
-GNinja \
78+
-DCMAKE_INSTALL_PREFIX=dependencies \
79+
-DCMAKE_PREFIX_PATH=dependencies \
80+
-DCMAKE_BUILD_TYPE=RelWithDebInfo
81+
cmake --build metkit-build -j -t install
82+
- name: Archive with permissions preserved
83+
run: tar --zstd -cpf files.tar.zst dependencies/ ecbuild/
84+
- name: Upload dependencies
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: deps
88+
path: files.tar.zst
89+
retention-days: 1
90+
build-wheels:
91+
needs: prepare-deps
92+
runs-on: ubuntu-latest
93+
strategy:
94+
matrix:
95+
python-version: ['3.11', '3.12', '3.13'] # eccodes does not yet support 3.14
96+
fail-fast: false # Continue running other versions if one fails
97+
98+
steps:
99+
- name: Set up Python ${{ matrix.python-version }}
100+
uses: actions/setup-python@v5
101+
with:
102+
python-version: ${{ matrix.python-version }}
103+
- name: Download dependencies
104+
uses: actions/download-artifact@v4
105+
with:
106+
name: deps
107+
- name: Extract with zstd
108+
run: tar --zstd -xpf files.tar.zst
109+
- name: Get fdb5
110+
uses: actions/checkout@v5
111+
with:
112+
repository: ecmwf/fdb
113+
path: fdb-src
114+
- name: Display Python version
115+
run: python --version
116+
- name: Install 'build'
117+
run: pip install -r fdb-src/requirements.txt
118+
- name: Build fdb
119+
run: |
120+
export PATH=$(pwd)/dependencies/bin:$PATH
121+
mkdir fdb-build
122+
cmake \
123+
-B fdb-build \
124+
-S fdb-src \
125+
-GNinja \
126+
-DCMAKE_INSTALL_PREFIX=dependencies \
127+
-DCMAKE_PREFIX_PATH=dependencies \
128+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
129+
-DENABLE_TOOLS=ON \
130+
-DENABLE_PYTHON_ZARR_INTERFACE=ON
131+
cmake --build fdb-build -j
132+
cd fdb-build
133+
ctest -j $(nproc) --output-on-failure
134+
- name: Upload wheel
135+
uses: actions/upload-artifact@v4
136+
with:
137+
name: z3fdb-wheel-py${{ matrix.python-version }}
138+
path: "fdb-build/z3fdb-*.whl"
139+
retention-days: 10
140+
if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,7 @@ install.sh
1010

1111
# LLVM debug
1212
launch.json
13+
14+
compile_commands.json
15+
16+
__pycache__/

‎CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ find_package(LibUUID QUIET)
7878

7979
find_package(DAOS QUIET)
8080

81+
8182
set(_default_dummy_daos ON)
8283
if(DAOS_FOUND)
8384
set(_default_dummy_daos OFF)
@@ -121,6 +122,15 @@ ecbuild_add_option( FEATURE SANDBOX
121122
DESCRIPTION "build the sandbox stuff"
122123
DEFAULT OFF )
123124

125+
ecbuild_add_option( FEATURE PYTHON_ZARR_INTERFACE
126+
DESCRIPTION "Build python Zarr interface layer"
127+
DEFAULT OFF
128+
REQUIRED_PACKAGES "NAME pybind11 VERSION 3.0.1" )
129+
if(HAVE_PYTHON_ZARR_INTERFACE)
130+
find_package(Python 3.11 COMPONENTS Interpreter Development REQUIRED)
131+
endif()
132+
133+
124134
### build the tools
125135

126136
ecbuild_add_option( FEATURE BUILD_TOOLS

‎README.rst‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,32 @@
11
===
2-
fdb
2+
FDB
33
===
44

55
|Licence|
66

7-
FDB (Fields DataBase) is a domain-specific object store developed at ECMWF for storing, indexing and retrieving GRIB data. Each GRIB message is stored as a field and indexed trough semantic metadata (i.e. physical variables such as temperature, pressure, ...).
8-
A set of fields can be retrieved specifying a request using a specific language developed for accessing MARS_ Archive
7+
FDB (Fields DataBase) is a domain-specific object store developed at ECMWF for
8+
storing, indexing and retrieving GRIB data. Each GRIB message is stored as a
9+
field and indexed trough semantic metadata (i.e. physical variables such as
10+
temperature, pressure, ...). A set of fields can be retrieved specifying a
11+
request using a specific language developed for accessing MARS_ Archive
912

10-
FDB exposes a C++ API as well as CLI tools_.
13+
FDB consists of several artefacts:
14+
15+
libfdb.so
16+
---------
17+
18+
In-process database with C++ API
19+
20+
fdb-tools
21+
---------
22+
23+
Commandline tools to interact with FDB trough CLI tools_
24+
25+
z3fdb
26+
-----
27+
28+
A python-zarr v3 store implementation that provides a virtual zarr store from
29+
FDB.
1130

1231

1332
Requirements
@@ -30,7 +49,8 @@ Installation
3049

3150
fdb employs an out-of-source build/install based on CMake.
3251

33-
Make sure ecbuild is installed and the ecbuild executable script is found ( ``which ecbuild`` ).
52+
Make sure ecbuild is installed and the ecbuild executable script is found (
53+
``which ecbuild`` ).
3454

3555
Now proceed with installation as follows:
3656
::
@@ -54,8 +74,11 @@ Now proceed with installation as follows:
5474
How to reference FDB5
5575
---------------------
5676

57-
Two publications, co-authored by Simon D. Smart, Tiago Quintino, Baudouin Raoult
58-
describe fdb architecture and have been presented at PASC'17 `A Scalable Object Store for Meteorological and Climate Data`_ and PASC'19 `A High-Performance Distributed Object-Store for Exascale Numerical Weather Prediction and Climate`_
77+
Two publications, co-authored by Simon D. Smart, Tiago Quintino, Baudouin
78+
Raoult describe fdb architecture and have been presented at PASC'17 `A Scalable
79+
Object Store for Meteorological and Climate Data`_ and PASC'19 `A
80+
High-Performance Distributed Object-Store for Exascale Numerical Weather
81+
Prediction and Climate`_
5982

6083
In the following the two BibTeX snippets:
6184
::
@@ -102,5 +125,4 @@ In the following the two BibTeX snippets:
102125
:target: https://github.com/ecmwf/fdb/blob/develop/LICENSE
103126
:alt: Apache Licence
104127

105-
.. _mars: docs/content/mars.rst
106128
.. _tools: docs/content/tools.rst

0 commit comments

Comments
 (0)