Skip to content

Commit 1587e63

Browse files
danovarodanovaro
committed
METK-160 added tool to derive a mars request from an input grib file with multiple messages
1 parent 7094899 commit 1587e63

File tree

6 files changed

+310
-0
lines changed

6 files changed

+310
-0
lines changed

‎src/tools/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ ecbuild_add_executable( TARGET odb-to-request
5151
NO_AS_NEEDED
5252
LIBS metkit eckit_option )
5353

54+
ecbuild_add_executable( TARGET grib-to-request
55+
SOURCES grib-to-request.cc
56+
CONDITION HAVE_GRIB AND HAVE_BUILD_TOOLS
57+
INCLUDES ${ECKIT_INCLUDE_DIRS}
58+
NO_AS_NEEDED
59+
LIBS metkit eckit_option )
60+
5461
ecbuild_add_executable( TARGET bufr-sanity-check
5562
SOURCES bufr-sanity-check.cc
5663
CONDITION HAVE_BUFR AND HAVE_BUILD_TOOLS

‎src/tools/grib-to-request.cc‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
#include <algorithm>
12+
#include <string>
13+
#include <vector>
14+
15+
#include "eckit/filesystem/PathName.h"
16+
#include "eckit/io/FileHandle.h"
17+
#include "eckit/log/JSON.h"
18+
#include "eckit/log/Log.h"
19+
#include "eckit/message/Message.h"
20+
#include "eckit/message/Reader.h"
21+
#include "eckit/option/CmdArgs.h"
22+
#include "eckit/option/Option.h"
23+
#include "eckit/option/SimpleOption.h"
24+
#include "eckit/runtime/Tool.h"
25+
#include "eckit/utils/StringTools.h"
26+
27+
#include "metkit/hypercube/HyperCube.h"
28+
#include "metkit/tool/MetkitTool.h"
29+
30+
using namespace metkit;
31+
using namespace metkit::mars;
32+
using namespace eckit;
33+
using namespace eckit::option;
34+
35+
//----------------------------------------------------------------------------------------------------------------------
36+
37+
class MarsRequestSetter : public eckit::message::MetadataGatherer {
38+
public: // methods
39+
40+
MarsRequestSetter(MarsRequest& request) : request_(request) {}
41+
42+
void setValue(const std::string& key, const std::string& value) override { request_.setValue(key, value); }
43+
void setValue(const std::string& key, long value) override { request_.setValue(key, value); }
44+
void setValue(const std::string& key, double value) override { request_.setValue(key, value); }
45+
46+
private: // members
47+
48+
MarsRequest& request_;
49+
};
50+
51+
//----------------------------------------------------------------------------------------------------------------------
52+
53+
class GribToRequestTool : public MetkitTool {
54+
public:
55+
56+
GribToRequestTool(int argc, char** argv) : MetkitTool(argc, argv) {
57+
options_.push_back(new SimpleOption<std::string>("verb", "Verb in the request, default = retrieve"));
58+
options_.push_back(
59+
new SimpleOption<std::string>("database", "add database keyword to requests, default = none"));
60+
options_.push_back(new SimpleOption<std::string>("source", "add source keyword to requests, default = none"));
61+
options_.push_back(new SimpleOption<std::string>("target", "add target keyword to requests, default = none"));
62+
options_.push_back(new SimpleOption<bool>("one",
63+
"Merge into one request, potentially describing more data "
64+
"fields than the ones in the input file, default = false"));
65+
options_.push_back(new SimpleOption<bool>("compact", "Merge into a small number of requests, default = false"));
66+
options_.push_back(new SimpleOption<bool>("json", "Format request in json, default = false"));
67+
}
68+
69+
virtual ~GribToRequestTool() {}
70+
71+
private: // methods
72+
73+
int minimumPositionalArguments() const { return 1; }
74+
75+
virtual void execute(const eckit::option::CmdArgs& args);
76+
77+
virtual void init(const CmdArgs& args);
78+
79+
virtual void usage(const std::string& tool) const;
80+
81+
private: // members
82+
83+
std::vector<PathName> paths_;
84+
std::string verb_ = "retrieve";
85+
std::string database_ = "";
86+
std::string source_ = "";
87+
std::string target_ = "";
88+
bool one_ = false;
89+
bool compact_ = false;
90+
bool json_ = false;
91+
};
92+
93+
//----------------------------------------------------------------------------------------------------------------------
94+
95+
void GribToRequestTool::init(const CmdArgs& args) {
96+
args.get("one", one_);
97+
args.get("compact", compact_);
98+
args.get("verb", verb_);
99+
args.get("database", database_);
100+
args.get("source", source_);
101+
args.get("target", target_);
102+
args.get("json", json_);
103+
104+
if (json_) {
105+
porcelain_ = true;
106+
}
107+
108+
if (one_ and compact_) {
109+
Log::error() << "Options --one and --compact are mutually exclusive" << std::endl;
110+
std::exit(1);
111+
}
112+
}
113+
114+
115+
void GribToRequestTool::usage(const std::string& tool) const {
116+
Log::info() << "Usage: " << tool << " [options] [request1] [request2] ..." << std::endl << std::endl;
117+
118+
Log::info() << "Examples:" << std::endl
119+
<< "=========" << std::endl
120+
<< std::endl
121+
<< tool << " --one --verb=retrieve data.grib" << std::endl
122+
<< tool << " --compact --verb=retrieve data.grib" << std::endl
123+
<< std::endl;
124+
}
125+
126+
static void toJSON(const std::vector<MarsRequest>& requests) {
127+
JSON j(Log::info());
128+
for (auto& r : requests) {
129+
r.json(j);
130+
}
131+
Log::info() << std::endl;
132+
}
133+
134+
static void toStdOut(const std::vector<MarsRequest>& requests) {
135+
for (auto& r : requests) {
136+
eckit::Log::info() << r << std::endl;
137+
}
138+
}
139+
140+
static void addKeyValue(std::vector<MarsRequest>& requests, const std::string& key, const std::string& value) {
141+
std::transform(requests.begin(), requests.end(), requests.begin(), [key, value](MarsRequest& r) -> MarsRequest {
142+
r.setValue(key, value);
143+
return r;
144+
});
145+
}
146+
147+
void GribToRequestTool::execute(const eckit::option::CmdArgs& args) {
148+
PathName inFile(args(0));
149+
150+
FileHandle dh(inFile);
151+
dh.openForRead();
152+
153+
eckit::message::Reader reader(dh, false);
154+
eckit::message::Message msg;
155+
156+
std::vector<MarsRequest> requests;
157+
158+
while ((msg = reader.next())) {
159+
MarsRequest r(verb_);
160+
MarsRequestSetter setter(r);
161+
162+
msg.getMetadata(setter);
163+
164+
if (one_ and requests.size()) {
165+
requests.back().merge(r);
166+
}
167+
else {
168+
requests.push_back(r);
169+
}
170+
}
171+
172+
if (compact_ && requests.size() > 1) {
173+
std::map<std::set<std::string>, std::vector<MarsRequest>> coherentRequests;
174+
175+
// split the requests into groups of requests with the same set of metadata (but potentially different values)
176+
for (auto& r : requests) {
177+
std::set<std::string> keys;
178+
for (const auto& p : r.parameters()) {
179+
keys.insert(p.name());
180+
}
181+
coherentRequests[keys].push_back(r);
182+
}
183+
requests.clear();
184+
185+
// compact each group of requests into a single request (if possible)
186+
for (auto& [keys, reqs] : coherentRequests) {
187+
if (reqs.size() == 1) { // it is a single field - return its request as is
188+
requests.push_back(reqs.front());
189+
continue;
190+
}
191+
192+
MarsRequest merged{reqs.front()};
193+
for (auto& r : reqs) {
194+
merged.merge(r);
195+
}
196+
if (merged.count() ==
197+
reqs.size()) { // the set of fields forms a full hypercube - return corresponding merged request
198+
requests.push_back(std::move(merged));
199+
continue;
200+
}
201+
202+
// sparse hypercube - we have to compute a set of compact requests describing the input fields
203+
std::vector<MarsRequest> compacted;
204+
metkit::hypercube::HyperCube h{merged};
205+
for (const auto& r : reqs) {
206+
h.clear(r);
207+
}
208+
for (const auto& r : h.requests()) {
209+
requests.push_back(r);
210+
}
211+
}
212+
}
213+
214+
if (not database_.empty()) {
215+
addKeyValue(requests, "database", database_);
216+
}
217+
if (StringTools::lower(verb_) == "archive") {
218+
addKeyValue(requests, "source", inFile);
219+
}
220+
if (not source_.empty()) {
221+
addKeyValue(requests, "source", source_);
222+
}
223+
if (not target_.empty()) {
224+
addKeyValue(requests, "target", target_);
225+
}
226+
if (json_) {
227+
toJSON(requests);
228+
}
229+
else {
230+
toStdOut(requests);
231+
}
232+
}
233+
234+
//----------------------------------------------------------------------------------------------------------------------
235+
236+
int main(int argc, char** argv) {
237+
GribToRequestTool tool(argc, argv);
238+
return tool.start();
239+
}

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,4 @@ ecbuild_add_test( TARGET metkit_test_c_compiled
108108
add_subdirectory(regressions)
109109
add_subdirectory(marsgen)
110110
add_subdirectory(mars2grib)
111+
add_subdirectory(tools)

‎tests/tools/CMakeLists.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
if ( HAVE_GRIB AND HAVE_BUILD_TOOLS )
3+
4+
list( APPEND metkit_tools
5+
grib-to-request
6+
)
7+
8+
foreach( _test ${metkit_tools} )
9+
ecbuild_configure_file( ${_test}.sh.in ${_test}.sh @ONLY )
10+
11+
ecbuild_add_test(
12+
TYPE SCRIPT
13+
COMMAND ${_test}.sh )
14+
15+
endforeach( )
16+
17+
endif()

‎tests/tools/d1.grib‎

23.4 KB
Binary file not shown.

‎tests/tools/grib-to-request.sh.in‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
3+
set -eux
4+
5+
yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
6+
die() { yell "$*"; exit 1; }
7+
try() { "$@" || die "Errored HERE => '$*'"; }
8+
9+
line_count() {
10+
[[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
11+
val=$(wc -l < out) && val=$((val + 0))
12+
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
13+
}
14+
15+
grep_count() {
16+
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
17+
val=$(grep -cE "$1" out)
18+
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
19+
}
20+
21+
22+
g2r="$<TARGET_FILE:grib-to-request>"
23+
24+
srcdir=@CMAKE_CURRENT_SOURCE_DIR@
25+
bindir=@CMAKE_CURRENT_BINARY_DIR@
26+
27+
export ECCODES_DEFINITION_PATH="@ECCODES_DEFINITION_PATH@"
28+
29+
$g2r $srcdir/d1.grib | tee out
30+
less out
31+
32+
line_count 16
33+
grep_count "step=0" 13
34+
grep_count "step=12" 3
35+
grep_count "timespan=fs" 3
36+
37+
$g2r --one $srcdir/d1.grib | tee out
38+
less out
39+
40+
line_count 1
41+
42+
$g2r --compact $srcdir/d1.grib | tee out
43+
less out
44+
45+
line_count 2
46+
grep_count "timespan=fs" 1

0 commit comments

Comments
 (0)