Skip to content

Commit 4e86ecc

Browse files
danovarodanovaro
committed
METK-160 added tool to derive a mars request from an input grib file with multiple messages
1 parent 37662f2 commit 4e86ecc

File tree

6 files changed

+309
-0
lines changed

6 files changed

+309
-0
lines changed

‎src/tools/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ ecbuild_add_executable( TARGET odb-to-request
5151
NO_AS_NEEDED
5252
LIBS metkit eckit_option )
5353

54+
ecbuild_add_executable( TARGET grib-to-request
55+
SOURCES grib-to-request.cc
56+
CONDITION HAVE_GRIB AND HAVE_BUILD_TOOLS
57+
INCLUDES ${ECKIT_INCLUDE_DIRS}
58+
NO_AS_NEEDED
59+
LIBS metkit eckit_option )
60+
5461
ecbuild_add_executable( TARGET bufr-sanity-check
5562
SOURCES bufr-sanity-check.cc
5663
CONDITION HAVE_BUFR AND HAVE_BUILD_TOOLS

‎src/tools/grib-to-request.cc‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
#include <algorithm>
12+
#include <map>
13+
#include <set>
14+
#include <string>
15+
#include <vector>
16+
17+
#include "eckit/filesystem/PathName.h"
18+
#include "eckit/io/FileHandle.h"
19+
#include "eckit/log/JSON.h"
20+
#include "eckit/log/Log.h"
21+
#include "eckit/message/Message.h"
22+
#include "eckit/message/Reader.h"
23+
#include "eckit/option/CmdArgs.h"
24+
#include "eckit/option/Option.h"
25+
#include "eckit/option/SimpleOption.h"
26+
#include "eckit/runtime/Tool.h"
27+
#include "eckit/utils/StringTools.h"
28+
29+
#include "metkit/hypercube/HyperCube.h"
30+
#include "metkit/tool/MetkitTool.h"
31+
32+
using namespace metkit;
33+
using namespace metkit::mars;
34+
using namespace eckit;
35+
using namespace eckit::option;
36+
37+
//----------------------------------------------------------------------------------------------------------------------
38+
39+
class MarsRequestSetter : public eckit::message::MetadataGatherer {
40+
public: // methods
41+
42+
MarsRequestSetter(MarsRequest& request) : request_(request) {}
43+
44+
void setValue(const std::string& key, const std::string& value) override { request_.setValue(key, value); }
45+
void setValue(const std::string& key, long value) override { request_.setValue(key, value); }
46+
void setValue(const std::string& key, double value) override { request_.setValue(key, value); }
47+
48+
private: // members
49+
50+
MarsRequest& request_;
51+
};
52+
53+
//----------------------------------------------------------------------------------------------------------------------
54+
55+
class GribToRequestTool : public MetkitTool {
56+
public:
57+
58+
GribToRequestTool(int argc, char** argv) : MetkitTool(argc, argv) {
59+
options_.push_back(new SimpleOption<std::string>("verb", "Verb in the request, default = retrieve"));
60+
options_.push_back(
61+
new SimpleOption<std::string>("database", "add database keyword to requests, default = none"));
62+
options_.push_back(new SimpleOption<std::string>("source", "add source keyword to requests, default = none"));
63+
options_.push_back(new SimpleOption<std::string>("target", "add target keyword to requests, default = none"));
64+
options_.push_back(new SimpleOption<bool>("one",
65+
"Merge into one request, potentially describing more data "
66+
"fields than the ones in the input file, default = false"));
67+
options_.push_back(new SimpleOption<bool>("compact", "Merge into a small number of requests, default = false"));
68+
options_.push_back(new SimpleOption<bool>("json", "Format request in json, default = false"));
69+
}
70+
71+
virtual ~GribToRequestTool() {}
72+
73+
private: // methods
74+
75+
int minimumPositionalArguments() const { return 1; }
76+
77+
virtual void execute(const eckit::option::CmdArgs& args);
78+
79+
virtual void init(const CmdArgs& args);
80+
81+
virtual void usage(const std::string& tool) const;
82+
83+
private: // members
84+
85+
std::vector<PathName> paths_;
86+
std::string verb_ = "retrieve";
87+
std::string database_ = "";
88+
std::string source_ = "";
89+
std::string target_ = "";
90+
bool one_ = false;
91+
bool compact_ = false;
92+
bool json_ = false;
93+
};
94+
95+
//----------------------------------------------------------------------------------------------------------------------
96+
97+
void GribToRequestTool::init(const CmdArgs& args) {
98+
args.get("one", one_);
99+
args.get("compact", compact_);
100+
args.get("verb", verb_);
101+
args.get("database", database_);
102+
args.get("source", source_);
103+
args.get("target", target_);
104+
args.get("json", json_);
105+
106+
if (json_) {
107+
porcelain_ = true;
108+
}
109+
110+
if (one_ and compact_) {
111+
Log::error() << "Options --one and --compact are mutually exclusive" << std::endl;
112+
std::exit(1);
113+
}
114+
}
115+
116+
117+
void GribToRequestTool::usage(const std::string& tool) const {
118+
Log::info() << "Usage: " << tool << " [options] [request1] [request2] ..." << std::endl << std::endl;
119+
120+
Log::info() << "Examples:" << std::endl
121+
<< "=========" << std::endl
122+
<< std::endl
123+
<< tool << " --one --verb=retrieve data.grib" << std::endl
124+
<< tool << " --compact --verb=retrieve data.grib" << std::endl
125+
<< std::endl;
126+
}
127+
128+
static void toJSON(const std::vector<MarsRequest>& requests) {
129+
JSON j(Log::info());
130+
for (auto& r : requests) {
131+
r.json(j);
132+
}
133+
Log::info() << std::endl;
134+
}
135+
136+
static void toStdOut(const std::vector<MarsRequest>& requests) {
137+
for (auto& r : requests) {
138+
eckit::Log::info() << r << std::endl;
139+
}
140+
}
141+
142+
static void addKeyValue(std::vector<MarsRequest>& requests, const std::string& key, const std::string& value) {
143+
std::transform(requests.begin(), requests.end(), requests.begin(), [key, value](MarsRequest& r) -> MarsRequest {
144+
r.setValue(key, value);
145+
return r;
146+
});
147+
}
148+
149+
void GribToRequestTool::execute(const eckit::option::CmdArgs& args) {
150+
PathName inFile(args(0));
151+
152+
FileHandle dh(inFile);
153+
dh.openForRead();
154+
155+
eckit::message::Reader reader(dh, false);
156+
eckit::message::Message msg;
157+
158+
std::vector<MarsRequest> requests;
159+
160+
while ((msg = reader.next())) {
161+
MarsRequest r(verb_);
162+
MarsRequestSetter setter(r);
163+
164+
msg.getMetadata(setter);
165+
166+
if (one_ and requests.size()) {
167+
requests.back().merge(r);
168+
}
169+
else {
170+
requests.push_back(r);
171+
}
172+
}
173+
174+
if (compact_ && requests.size() > 1) {
175+
std::map<std::set<std::string>, std::vector<MarsRequest>> coherentRequests;
176+
177+
// split the requests into groups of requests with the same set of metadata (but potentially different values)
178+
for (const auto& r : requests) {
179+
std::set<std::string> keys;
180+
for (const auto& p : r.parameters()) {
181+
keys.insert(p.name());
182+
}
183+
coherentRequests[keys].push_back(r);
184+
}
185+
requests.clear();
186+
187+
// compact each group of requests into a single request (if possible)
188+
for (const auto& [keys, reqs] : coherentRequests) {
189+
if (reqs.size() == 1) { // it is a single field - return its request as is
190+
requests.push_back(reqs.front());
191+
continue;
192+
}
193+
194+
MarsRequest merged{reqs.front()};
195+
for (size_t i = 1; i < reqs.size(); ++i) {
196+
merged.merge(reqs[i]);
197+
}
198+
if (merged.count() ==
199+
reqs.size()) { // the set of fields forms a full hypercube - return corresponding merged request
200+
requests.push_back(std::move(merged));
201+
continue;
202+
}
203+
204+
// sparse hypercube - we have to compute a set of compact requests describing the input fields
205+
metkit::hypercube::HyperCube h{merged};
206+
for (const auto& r : reqs) {
207+
h.clear(r);
208+
}
209+
for (const auto& r : h.requests()) {
210+
requests.push_back(r);
211+
}
212+
}
213+
}
214+
215+
if (not database_.empty()) {
216+
addKeyValue(requests, "database", database_);
217+
}
218+
if (StringTools::lower(verb_) == "archive") {
219+
addKeyValue(requests, "source", inFile);
220+
}
221+
if (not source_.empty()) {
222+
addKeyValue(requests, "source", source_);
223+
}
224+
if (not target_.empty()) {
225+
addKeyValue(requests, "target", target_);
226+
}
227+
if (json_) {
228+
toJSON(requests);
229+
}
230+
else {
231+
toStdOut(requests);
232+
}
233+
}
234+
235+
//----------------------------------------------------------------------------------------------------------------------
236+
237+
int main(int argc, char** argv) {
238+
GribToRequestTool tool(argc, argv);
239+
return tool.start();
240+
}

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,4 @@ ecbuild_add_test( TARGET metkit_test_c_compiled
108108
add_subdirectory(regressions)
109109
add_subdirectory(marsgen)
110110
add_subdirectory(mars2grib)
111+
add_subdirectory(tools)

‎tests/tools/CMakeLists.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
if ( HAVE_GRIB AND HAVE_BUILD_TOOLS )
3+
4+
list( APPEND metkit_tools
5+
grib-to-request
6+
)
7+
8+
foreach( _test ${metkit_tools} )
9+
ecbuild_configure_file( ${_test}.sh.in ${_test}.sh @ONLY )
10+
11+
ecbuild_add_test(
12+
TYPE SCRIPT
13+
COMMAND ${_test}.sh )
14+
15+
endforeach( )
16+
17+
endif()

‎tests/tools/d1.grib‎

23.4 KB
Binary file not shown.

‎tests/tools/grib-to-request.sh.in‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env bash
2+
3+
set -eux
4+
5+
yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
6+
die() { yell "$*"; exit 1; }
7+
8+
line_count() {
9+
[[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
10+
val=$(wc -l < out) && val=$((val + 0))
11+
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
12+
}
13+
14+
grep_count() {
15+
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
16+
val=$(grep -cE "$1" out)
17+
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
18+
}
19+
20+
21+
g2r="$<TARGET_FILE:grib-to-request>"
22+
23+
srcdir=@CMAKE_CURRENT_SOURCE_DIR@
24+
bindir=@CMAKE_CURRENT_BINARY_DIR@
25+
26+
export ECCODES_DEFINITION_PATH="@ECCODES_DEFINITION_PATH@"
27+
28+
$g2r $srcdir/d1.grib | tee out
29+
30+
line_count 16
31+
grep_count "step=0" 13
32+
grep_count "step=12" 3
33+
grep_count "timespan=fs" 3
34+
35+
36+
$g2r --one $srcdir/d1.grib | tee out
37+
38+
line_count 1
39+
40+
41+
$g2r --compact $srcdir/d1.grib | tee out
42+
43+
line_count 2
44+
grep_count "timespan=fs" 1

0 commit comments

Comments
 (0)