Skip to content

Commit d3ef22a

Browse files
danovarodanovaro
committed
Merge branch 'hotfix/1.28.7'
2 parents 0e69ea0 + 4e37082 commit d3ef22a

File tree

5 files changed

+199
-10
lines changed

5 files changed

+199
-10
lines changed

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.28.6
1+
1.28.7

‎src/eckit/utils/RLE.cc‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
#include <cstddef>
1212
#include <iterator>
13+
#include <optional>
1314
#include <vector>
1415

1516
#include "eckit/log/Log.h"
@@ -43,9 +44,10 @@ dummy_iterator<T> make_dummy(T*) {
4344
return dummy_iterator<T>();
4445
}
4546

46-
4747
template <class T, class U>
48-
long long RLEencode2(T first, T last, U output, long long maxLoop) {
48+
long long RLEencode2timeout(T first, T last, U output, long long maxLoop,
49+
std::optional<EncodingClock::time_point> deadline) {
50+
4951
long long x = 0;
5052
long long m = 0;
5153
long long j = 0;
@@ -77,15 +79,19 @@ long long RLEencode2(T first, T last, U output, long long maxLoop) {
7779
m = a;
7880
x = n;
7981
j = i;
80-
if (m > enough)
82+
if (m > enough || (deadline && EncodingClock::now() > deadline.value()))
8183
goto stop;
8284
}
8385
}
86+
87+
if (deadline && EncodingClock::now() > deadline.value()) {
88+
goto stop;
89+
}
8490
}
8591
stop:
8692

8793
if (m == 0) {
88-
copy(first, last, output);
94+
std::copy(first, last, output);
8995
return last - first;
9096
}
9197
else {
@@ -97,26 +103,45 @@ long long RLEencode2(T first, T last, U output, long long maxLoop) {
97103
other += x;
98104
}
99105

100-
long long n = RLEencode2(first, from, output, maxLoop);
106+
long long n = RLEencode2timeout(first, from, output, maxLoop, deadline);
101107

102108
if (k > 1) {
103109
*output++ = -k;
104110
n++;
105-
int m = RLEencode2(from, from + x, make_dummy((typename std::iterator_traits<T>::value_type*)(0)), maxLoop);
111+
int m = RLEencode2timeout(from, from + x, make_dummy((typename std::iterator_traits<T>::value_type*)(0)),
112+
maxLoop, deadline);
106113

107114
if (m > 1) {
108115
*output++ = -m;
109116
n++;
110117
}
111118
}
112119

113-
n += RLEencode2(from, from + x, output, maxLoop);
114-
n += RLEencode2(from + k * x, last, output, maxLoop);
120+
n += RLEencode2timeout(from, from + x, output, maxLoop, deadline);
121+
n += RLEencode2timeout(from + k * x, last, output, maxLoop, deadline);
115122

116123
return n;
117124
}
118125
}
119126

127+
template <class T, class U>
128+
long long RLEencode2(T first, T last, U output, long long maxLoop) {
129+
return RLEencode2timeout(first, last, output, maxLoop, std::optional<EncodingClock::time_point>{});
130+
}
131+
132+
template <class T, class U>
133+
long long RLEencode2(T first, T last, U output, long long maxLoop, const EncodingClock::duration timelimit) {
134+
return RLEencode2timeout(first, last, output, maxLoop,
135+
std::optional<EncodingClock::time_point>{EncodingClock::now() + timelimit});
136+
}
137+
138+
template <class InputIterator, class OutputIterator>
139+
long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop);
140+
141+
template <class InputIterator, class OutputIterator>
142+
long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop,
143+
const EncodingClock::duration timelimit);
144+
120145
template <class T, class U>
121146
void RLEdecode2(T first, T last, U output) {
122147
while (first != last) {

‎src/eckit/utils/RLE.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,26 @@
1414
#ifndef eckit_RLE_h
1515
#define eckit_RLE_h
1616

17-
17+
#include <chrono>
1818
#include <iosfwd>
1919

2020
//-----------------------------------------------------------------------------
2121

2222
namespace eckit {
2323

24+
using EncodingClock = std::chrono::steady_clock;
25+
2426
//-----------------------------------------------------------------------------
2527

2628
class Stream;
2729

2830
template <class InputIterator, class OutputIterator>
2931
long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop);
3032

33+
template <class InputIterator, class OutputIterator>
34+
long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop,
35+
const EncodingClock::duration timelimit);
36+
3137
template <class InputIterator, class OutputIterator>
3238
void RLEdecode2(InputIterator first, InputIterator last, OutputIterator result);
3339

‎tests/utils/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ ecbuild_add_test( TARGET eckit_test_utils_rendezvoushash
3737
SOURCES test_rendezvoushash.cc
3838
LIBS eckit )
3939

40+
ecbuild_add_test( TARGET eckit_test_utils_rle
41+
SOURCES test_rle.cc
42+
LIBS eckit )
43+
4044
ecbuild_add_test( TARGET eckit_test_utils_compressor
4145
SOURCES test_compressor.cc
4246
LIBS eckit )

‎tests/utils/test_rle.cc‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
#include <iostream>
12+
#include <limits>
13+
#include <memory>
14+
#include <random>
15+
#include <set>
16+
#include <vector>
17+
18+
#include "eckit/config/LibEcKit.h"
19+
#include "eckit/log/Timer.h"
20+
#include "eckit/utils/RLE.h"
21+
22+
#include "eckit/testing/Test.h"
23+
24+
using namespace std;
25+
using namespace eckit;
26+
using namespace eckit::testing;
27+
28+
namespace eckit::test {
29+
30+
//----------------------------------------------------------------------------------------------------------------------
31+
32+
void test(const std::vector<long>& in, EncodingClock::duration timeout = std::chrono::milliseconds(2),
33+
long expectedSize = -1) {
34+
35+
std::vector<long> out;
36+
std::vector<long> outTimeout;
37+
int maxLoop = 5000;
38+
39+
long long n;
40+
long long t;
41+
{
42+
eckit::Timer timer;
43+
n = RLEencode2(in.begin(), in.end(), std::back_inserter(out), maxLoop);
44+
LOG_DEBUG_LIB(LibEcKit) << "no_timeout - time elapsed: " << timer.elapsed() << " - output size: " << out.size()
45+
<< std::endl;
46+
}
47+
{
48+
eckit::Timer timer;
49+
t = RLEencode2(in.begin(), in.end(), std::back_inserter(outTimeout), maxLoop, timeout);
50+
LOG_DEBUG_LIB(LibEcKit) << "timeout " << std::chrono::duration_cast<std::chrono::milliseconds>(timeout).count()
51+
<< "ms - time elapsed: " << timer.elapsed() << " - output size: " << outTimeout.size()
52+
<< std::endl;
53+
}
54+
55+
if (expectedSize > 0) {
56+
EXPECT_EQUAL(expectedSize, n);
57+
EXPECT_EQUAL(expectedSize, t);
58+
EXPECT_EQUAL(out.size(), outTimeout.size());
59+
for (size_t i = 0; i < out.size(); i++) {
60+
EXPECT_EQUAL(out[i], outTimeout[i]);
61+
}
62+
}
63+
64+
std::vector<long> decoded;
65+
RLEdecode2(out.begin(), out.end(), std::back_inserter(decoded));
66+
EXPECT_EQUAL(in.size(), decoded.size());
67+
for (size_t i = 0; i < in.size(); i++) {
68+
EXPECT_EQUAL(in[i], decoded[i]);
69+
}
70+
71+
std::vector<long> decodedTimeout;
72+
RLEdecode2(outTimeout.begin(), outTimeout.end(), std::back_inserter(decodedTimeout));
73+
EXPECT_EQUAL(in.size(), decodedTimeout.size());
74+
for (size_t i = 0; i < in.size(); i++) {
75+
EXPECT_EQUAL(in[i], decodedTimeout[i]);
76+
}
77+
}
78+
79+
CASE("single") {
80+
std::vector<long> in = {4};
81+
82+
test(in, std::chrono::seconds(1), 1);
83+
}
84+
85+
CASE("identical") {
86+
size_t size = 1000;
87+
std::vector<long> in;
88+
in.reserve(size);
89+
for (size_t i = 0; i < size; i++) {
90+
in.push_back(4);
91+
}
92+
93+
test(in, std::chrono::seconds(1), 2); /// we expect [-1000,4]
94+
}
95+
96+
CASE("interleaved") {
97+
size_t size = 1000;
98+
std::vector<long> in;
99+
in.reserve(size);
100+
for (size_t i = 0; i < size; i++) {
101+
in.push_back(i % 2);
102+
}
103+
104+
test(in, std::chrono::seconds(1), 4); /// we expect [-500,-2,0,1]
105+
}
106+
107+
CASE("pattern") {
108+
size_t size = 100000;
109+
std::vector<long> in;
110+
in.reserve(size);
111+
for (size_t i = 0; i < size; i++) {
112+
in.push_back(std::ceil(std::sqrt(i)));
113+
}
114+
115+
test(in);
116+
}
117+
118+
119+
void randTest(int limit) {
120+
std::random_device rd; // a seed source for the random number engine
121+
std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd()
122+
std::uniform_int_distribution<> distrib(1, std::numeric_limits<int>::max());
123+
124+
size_t size = 10000000;
125+
std::vector<long> in;
126+
in.reserve(size);
127+
for (size_t i = 0; i < size; i++) {
128+
in.push_back(distrib(gen));
129+
}
130+
test(in);
131+
}
132+
133+
CASE("randMaxInt") {
134+
randTest(std::numeric_limits<int>::max());
135+
}
136+
137+
CASE("rand100") {
138+
randTest(100);
139+
}
140+
141+
CASE("rand10") {
142+
randTest(10);
143+
}
144+
145+
CASE("rand2") {
146+
randTest(2);
147+
}
148+
//----------------------------------------------------------------------------------------------------------------------
149+
150+
} // namespace eckit::test
151+
152+
int main(int argc, char* argv[]) {
153+
return run_tests(argc, argv);
154+
}

0 commit comments

Comments
 (0)