Skip to content

Commit f9a9877

Browse files
blootsvoetsblootsvoets
authored
Merge pull request #35 from RADAR-base/release-0.5.2
Release 0.5.2
2 parents c9cd842 + bce3d7f commit f9a9877

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+636
-170
lines changed

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/R
88

99
This package is available as docker image [`radarbase/radar-hdfs-restructure`](https://hub.docker.com/r/radarbase/radar-hdfs-restructure). The entrypoint of the image is the current application. So in all of the commands listed in usage, replace `radar-hdfs-restructure` with for example:
1010
```shell
11-
docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.5.1 -n hdfs-namenode -o /output /myTopic
11+
docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.5.2 -n hdfs-namenode -o /output /myTopic
1212
```
1313
if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
1414

@@ -23,7 +23,7 @@ This package requires at least Java JDK 8. Build the distribution with
2323
and install the package into `/usr/local` with for example
2424
```shell
2525
sudo mkdir -p /usr/local
26-
sudo tar -xzf build/distributions/radar-hdfs-restructure-0.5.1.tar.gz -C /usr/local --strip-components=1
26+
sudo tar -xzf build/distributions/radar-hdfs-restructure-0.5.2.tar.gz -C /usr/local --strip-components=1
2727
```
2828

2929
Now the `radar-hdfs-restructure` command should be available.
@@ -58,6 +58,8 @@ radar-hdfs-restructure --compression gzip --nameservice <hdfs_node> --output-di
5858

5959
By default, files records are not deduplicated after writing. To enable this behaviour, specify the option `--deduplicate` or `-d`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.
6060

61+
To set the output user ID and group ID, specify the `-p local-uid=123` and `-p local-gid=12` properties.
62+
6163
## Extending the connector
6264

6365
To implement alternative storage paths, storage drivers or storage formats, put your custom JAR in

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ plugins {
77
}
88

99
group 'org.radarcns'
10-
version '0.5.1'
10+
version '0.5.2'
1111
mainClassName = 'org.radarcns.hdfs.Application'
1212

1313
sourceCompatibility = '1.8'

‎src/main/java/org/radarcns/hdfs/Application.java‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,36 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs;
218

19+
import static org.radarcns.hdfs.util.ProgressBar.formatTime;
20+
import static org.radarcns.hdfs.util.commandline.CommandLineArgs.nonNullOrDefault;
21+
322
import com.beust.jcommander.JCommander;
423
import com.beust.jcommander.ParameterException;
24+
import java.io.IOException;
25+
import java.io.UncheckedIOException;
26+
import java.nio.file.Files;
27+
import java.text.SimpleDateFormat;
28+
import java.time.Duration;
29+
import java.time.Instant;
30+
import java.util.Date;
31+
import java.util.HashMap;
32+
import java.util.List;
33+
import java.util.Map;
534
import org.radarcns.hdfs.accounting.Accountant;
635
import org.radarcns.hdfs.config.HdfsSettings;
736
import org.radarcns.hdfs.config.RestructureSettings;
@@ -17,20 +46,6 @@
1746
import org.slf4j.Logger;
1847
import org.slf4j.LoggerFactory;
1948

20-
import java.io.IOException;
21-
import java.io.UncheckedIOException;
22-
import java.nio.file.Files;
23-
import java.text.SimpleDateFormat;
24-
import java.time.Duration;
25-
import java.time.Instant;
26-
import java.util.Date;
27-
import java.util.HashMap;
28-
import java.util.List;
29-
import java.util.Map;
30-
31-
import static org.radarcns.hdfs.util.ProgressBar.formatTime;
32-
import static org.radarcns.hdfs.util.commandline.CommandLineArgs.nonNullOrDefault;
33-
3449
/** Main application. */
3550
public class Application implements FileStoreFactory {
3651
private static final Logger logger = LoggerFactory.getLogger(Application.class);

‎src/main/java/org/radarcns/hdfs/FileStoreFactory.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,22 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs;
218

19+
import java.io.IOException;
320
import org.radarcns.hdfs.accounting.Accountant;
421
import org.radarcns.hdfs.config.HdfsSettings;
522
import org.radarcns.hdfs.config.RestructureSettings;
@@ -8,8 +25,6 @@
825
import org.radarcns.hdfs.data.RecordConverterFactory;
926
import org.radarcns.hdfs.data.StorageDriver;
1027

11-
import java.io.IOException;
12-
1328
/** Factory for all factory classes and settings. */
1429
public interface FileStoreFactory {
1530
FileCacheStore newFileCacheStore(Accountant accountant) throws IOException;

‎src/main/java/org/radarcns/hdfs/ObservationKeyPathFactory.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,25 @@
1-
package org.radarcns.hdfs;
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
216

3-
import org.apache.avro.generic.GenericRecord;
17+
package org.radarcns.hdfs;
418

519
import java.nio.file.Path;
620
import java.nio.file.Paths;
721
import java.time.Instant;
22+
import org.apache.avro.generic.GenericRecord;
823

924
public class ObservationKeyPathFactory extends RecordPathFactory {
1025
@Override

‎src/main/java/org/radarcns/hdfs/Plugin.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs;
218

319
import java.io.IOException;

‎src/main/java/org/radarcns/hdfs/RadarHdfsRestructure.java‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,23 @@
1616

1717
package org.radarcns.hdfs;
1818

19+
import static org.radarcns.hdfs.util.ProgressBar.formatTime;
20+
1921
import com.fasterxml.jackson.databind.JsonMappingException;
22+
import java.io.IOException;
23+
import java.io.UncheckedIOException;
24+
import java.text.NumberFormat;
25+
import java.time.Duration;
26+
import java.time.Instant;
27+
import java.util.Comparator;
28+
import java.util.List;
29+
import java.util.concurrent.ExecutorService;
30+
import java.util.concurrent.Executors;
31+
import java.util.concurrent.ThreadLocalRandom;
32+
import java.util.concurrent.TimeUnit;
33+
import java.util.concurrent.atomic.LongAdder;
34+
import java.util.stream.Collectors;
35+
import java.util.stream.Stream;
2036
import org.apache.avro.file.DataFileReader;
2137
import org.apache.avro.generic.GenericDatumReader;
2238
import org.apache.avro.generic.GenericRecord;
@@ -37,25 +53,6 @@
3753
import org.slf4j.Logger;
3854
import org.slf4j.LoggerFactory;
3955

40-
import java.io.IOException;
41-
import java.io.UncheckedIOException;
42-
import java.text.NumberFormat;
43-
import java.text.SimpleDateFormat;
44-
import java.time.Duration;
45-
import java.time.Instant;
46-
import java.util.Comparator;
47-
import java.util.List;
48-
import java.util.TimeZone;
49-
import java.util.concurrent.ExecutorService;
50-
import java.util.concurrent.Executors;
51-
import java.util.concurrent.ThreadLocalRandom;
52-
import java.util.concurrent.TimeUnit;
53-
import java.util.concurrent.atomic.LongAdder;
54-
import java.util.stream.Collectors;
55-
import java.util.stream.Stream;
56-
57-
import static org.radarcns.hdfs.util.ProgressBar.formatTime;
58-
5956
public class RadarHdfsRestructure {
6057
private static final Logger logger = LoggerFactory.getLogger(RadarHdfsRestructure.class);
6158

‎src/main/java/org/radarcns/hdfs/RecordPathFactory.java‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,31 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs;
218

3-
import org.apache.avro.Schema;
4-
import org.apache.avro.generic.GenericRecord;
5-
import org.slf4j.Logger;
6-
import org.slf4j.LoggerFactory;
19+
import static java.time.ZoneOffset.UTC;
720

821
import java.nio.file.Path;
922
import java.time.Instant;
1023
import java.time.format.DateTimeFormatter;
1124
import java.util.regex.Pattern;
12-
13-
import static java.time.ZoneOffset.UTC;
25+
import org.apache.avro.Schema;
26+
import org.apache.avro.generic.GenericRecord;
27+
import org.slf4j.Logger;
28+
import org.slf4j.LoggerFactory;
1429

1530
public abstract class RecordPathFactory implements Plugin {
1631
private static final Logger logger = LoggerFactory.getLogger(RecordPathFactory.class);

‎src/main/java/org/radarcns/hdfs/accounting/Accountant.java‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,28 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs.accounting;
218

19+
import java.io.Closeable;
20+
import java.io.Flushable;
21+
import java.io.IOException;
22+
import java.nio.file.Path;
23+
import java.nio.file.Paths;
24+
import java.util.HashMap;
25+
import java.util.Map;
326
import org.radarcns.hdfs.FileStoreFactory;
427
import org.radarcns.hdfs.config.RestructureSettings;
528
import org.radarcns.hdfs.data.StorageDriver;
@@ -9,14 +32,6 @@
932
import org.slf4j.Logger;
1033
import org.slf4j.LoggerFactory;
1134

12-
import java.io.Closeable;
13-
import java.io.Flushable;
14-
import java.io.IOException;
15-
import java.nio.file.Path;
16-
import java.nio.file.Paths;
17-
import java.util.HashMap;
18-
import java.util.Map;
19-
2035
public class Accountant implements Flushable, Closeable {
2136
private static final Logger logger = LoggerFactory.getLogger(Accountant.class);
2237

‎src/main/java/org/radarcns/hdfs/accounting/Bin.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,23 @@
1+
/*
2+
* Copyright 2018 The Hyve
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
package org.radarcns.hdfs.accounting;
218

3-
import javax.annotation.Nonnull;
419
import java.util.Objects;
20+
import javax.annotation.Nonnull;
521

622
public class Bin {
723
private final String topic;

0 commit comments

Comments
 (0)