Skip to content

Commit 1938b64

Browse files
blootsvoetsblootsvoets
authored
Merge pull request #24 from RADAR-base/release-0.2.1
Release 0.2.1
2 parents a3bbebf + f2ff022 commit 1938b64

27 files changed

+594
-101
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src
4040

4141
RUN ./gradlew jar
4242

43-
FROM confluentinc/cp-kafka-connect-base:5.0.0
43+
FROM confluentinc/cp-kafka-connect-base:5.1.0
4444

4545
MAINTAINER Joris Borgdorff <[email protected]>
4646

‎README.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@ your Fitbit App client ID and client secret. The following tables shows the poss
3939
<tr>
4040
<td>rest.source.base.url</td></td><td>Base URL for REST source connector.</td></td><td>string</td></td><td></td></td><td></td></td><td>high</td></td></tr>
4141
<tr>
42+
<td>rest.source.destination.topics</td></td><td>The list of destination topics for the REST source connector.</td></td><td>list</td></td><td>""</td></td><td></td></td><td>high</td></td></tr>
43+
<tr>
44+
<td>rest.source.topic.selector</td></td><td>The topic selector class for REST source connector.</td></td><td>class</td></td><td>org.radarbase.connect.rest.selector.SimpleTopicSelector</td></td><td>Class extending org.radarbase.connect.rest.selector.TopicSelector</td></td><td>high</td></td></tr>
45+
<tr>
46+
<td>rest.source.payload.converter.class</td></td><td>Class to be used to convert messages from REST calls to SourceRecords</td></td><td>class</td></td><td>org.radarbase.connect.rest.converter.StringPayloadConverter</td></td><td>Class extending org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter</td></td><td>low</td></td></tr>
47+
<tr>
4248
<td>rest.source.request.generator.class</td></td><td>Class to be used to generate REST requests</td></td><td>class</td></td><td>org.radarbase.connect.rest.single.SingleRequestGenerator</td></td><td>Class extending org.radarbase.connect.rest.request.RequestGenerator</td></td><td>low</td></td></tr>
4349
<tr>
4450
<td>fitbit.users</td></td><td>The user ID of Fitbit users to include in polling, separated by commas. Non existing user names will be ignored. If empty, all users in the user directory will be used.</td></td><td>list</td></td><td>""</td></td><td></td></td><td>high</td></td></tr>
@@ -47,10 +53,14 @@ your Fitbit App client ID and client secret. The following tables shows the poss
4753
<tr>
4854
<td>fitbit.api.secret</td></td><td>Secret for the Fitbit API client set in fitbit.api.client.</td></td><td>password</td></td><td></td></td><td></td></td><td>high</td></td></tr>
4955
<tr>
50-
<td>fitbit.user.repository.class</td></td><td>Class for managing users and authentication.</td></td><td>class</td></td><td>org.radarbase.connect.rest.fitbit.user.YamlFitbitUserRepository</td></td><td>Class extending org.radarbase.connect.rest.fitbit.user.FitbitUserRepository</td></td><td>medium</td></td></tr>
56+
<td>fitbit.api.intraday</td></td><td>Set to true if the client has permissions to Fitbit Intraday API, false otherwise.</td></td><td>boolean</td></td><td>true</td></td><td></td></td><td>medium</td></td></tr>
57+
<tr>
58+
<td>fitbit.user.repository.class</td></td><td>Class for managing users and authentication.</td></td><td>class</td></td><td>org.radarbase.connect.rest.fitbit.user.YamlUserRepository</td></td><td>Class extending org.radarbase.connect.rest.fitbit.user.UserRepository</td></td><td>medium</td></td></tr>
5159
<tr>
5260
<td>fitbit.user.dir</td></td><td>Directory containing Fitbit user information and credentials. Only used if a file-based user repository is configured.</td></td><td>string</td></td><td>/var/lib/kafka-connect-fitbit-source/users</td></td><td></td></td><td>low</td></td></tr>
5361
<tr>
62+
<td>fitbit.user.repository.url</td></td><td>URL for webservice containing user credentials. Only used if a webservice-based user repository is configured.</td></td><td>string</td></td><td>""</td></td><td></td></td><td>low</td></td></tr>
63+
<tr>
5464
<td>fitbit.max.users.per.poll</td></td><td>Maximum number of users to query in a single poll operation. Decrease this if memory constrains are pressing.</td></td><td>int</td></td><td>100</td></td><td>[1,...]</td></td><td>low</td></td></tr>
5565
<tr>
5666
<td>fitbit.intraday.steps.topic</td></td><td>Topic for Fitbit intraday steps</td></td><td>string</td></td><td>connect_fitbit_intraday_steps</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
@@ -61,7 +71,9 @@ your Fitbit App client ID and client secret. The following tables shows the poss
6171
<tr>
6272
<td>fitbit.sleep.classic.topic</td></td><td>Topic for Fitbit sleep classic data</td></td><td>string</td></td><td>connect_fitbit_sleep_classic</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
6373
<tr>
64-
<td>fitbit.time.zone.topic</td></td><td>Topic for Fitbit profile timezone</td></td><td>string</td></td><td>connect_fitbit_time_zone</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
74+
<td>fitbit.time.zone.topic</td></td><td>Topic for Fitbit profile time zone</td></td><td>string</td></td><td>connect_fitbit_time_zone</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
75+
<tr>
76+
<td>fitbit.activity.log.topic</td></td><td>Topic for Fitbit activity log.</td></td><td>string</td></td><td>connect_fitbit_activity_log</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
6577
</tbody></table>
6678

6779
Now you can run a full Kafka stack using

‎build.gradle‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ description = 'kafka-connect-rest-source'
22

33
subprojects {
44
ext {
5-
kafkaVersion = '2.0.0'
6-
confluentVersion = '5.0.0'
7-
jacksonVersion = '2.8.9'
5+
kafkaVersion = '2.1.0-cp1'
6+
confluentVersion = '5.1.0'
7+
jacksonVersion = '2.9.8'
88
}
99

1010
apply plugin: 'java'
1111
apply plugin: 'java-library'
1212

1313
group = 'org.radarcns'
14-
version = '0.2.0'
14+
version = '0.2.1'
1515

1616
sourceCompatibility = 1.8
1717
targetCompatibility = 1.8

‎docker-compose.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ services:
99
# Zookeeper Cluster #
1010
#---------------------------------------------------------------------------#
1111
zookeeper-1:
12-
image: confluentinc/cp-zookeeper:5.0.0
12+
image: confluentinc/cp-zookeeper:5.1.0
1313
environment:
1414
ZOOKEEPER_SERVER_ID: 1
1515
ZOOKEEPER_CLIENT_PORT: 2181
@@ -19,7 +19,7 @@ services:
1919
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888
2020

2121
zookeeper-2:
22-
image: confluentinc/cp-zookeeper:5.0.0
22+
image: confluentinc/cp-zookeeper:5.1.0
2323
environment:
2424
ZOOKEEPER_SERVER_ID: 2
2525
ZOOKEEPER_CLIENT_PORT: 2181
@@ -29,7 +29,7 @@ services:
2929
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888
3030

3131
zookeeper-3:
32-
image: confluentinc/cp-zookeeper:5.0.0
32+
image: confluentinc/cp-zookeeper:5.1.0
3333
environment:
3434
ZOOKEEPER_SERVER_ID: 3
3535
ZOOKEEPER_CLIENT_PORT: 2181
@@ -42,7 +42,7 @@ services:
4242
# Kafka Cluster #
4343
#---------------------------------------------------------------------------#
4444
kafka-1:
45-
image: confluentinc/cp-kafka:5.0.0
45+
image: confluentinc/cp-kafka:5.1.0
4646
depends_on:
4747
- zookeeper-1
4848
- zookeeper-2
@@ -61,7 +61,7 @@ services:
6161
KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
6262

6363
kafka-2:
64-
image: confluentinc/cp-kafka:5.0.0
64+
image: confluentinc/cp-kafka:5.1.0
6565
depends_on:
6666
- zookeeper-1
6767
- zookeeper-2
@@ -80,7 +80,7 @@ services:
8080
KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
8181

8282
kafka-3:
83-
image: confluentinc/cp-kafka:5.0.0
83+
image: confluentinc/cp-kafka:5.1.0
8484
depends_on:
8585
- zookeeper-1
8686
- zookeeper-2
@@ -102,7 +102,7 @@ services:
102102
# Schema Registry #
103103
#---------------------------------------------------------------------------#
104104
schema-registry-1:
105-
image: confluentinc/cp-schema-registry:5.0.0
105+
image: confluentinc/cp-schema-registry:5.1.0
106106
depends_on:
107107
- zookeeper-1
108108
- zookeeper-2
@@ -124,7 +124,7 @@ services:
124124
# REST proxy #
125125
#---------------------------------------------------------------------------#
126126
rest-proxy-1:
127-
image: confluentinc/cp-kafka-rest:5.0.0
127+
image: confluentinc/cp-kafka-rest:5.1.0
128128
depends_on:
129129
- zookeeper-1
130130
- zookeeper-2

‎kafka-connect-fitbit-source/build.gradle‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
1+
repositories {
2+
maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' }
3+
}
4+
15
dependencies {
26
api project(':kafka-connect-rest-source')
37
api group: 'io.confluent', name: 'kafka-connect-avro-converter', version: confluentVersion
4-
api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.3.7-SNAPSHOT'
8+
api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.4.4-SNAPSHOT'
59

610
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion
711
implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion
@@ -10,8 +14,8 @@ dependencies {
1014
compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion
1115
compileOnly group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion
1216

13-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.2.0'
14-
testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.2.0'
17+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.3.2'
18+
testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.3.2'
1519
testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25'
1620
testImplementation group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion
1721
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.nio.charset.StandardCharsets;
2424
import java.nio.file.Path;
2525
import java.nio.file.Paths;
26+
import java.time.Duration;
2627
import java.util.Base64;
2728
import java.util.Collections;
2829
import java.util.List;
@@ -64,6 +65,11 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
6465
private static final String FITBIT_USER_REPOSITORY_DOC = "Class for managing users and authentication.";
6566
private static final String FITBIT_USER_REPOSITORY_DISPLAY = "User repository class";
6667

68+
public static final String FITBIT_API_INTRADAY_ACCESS_CONFIG = "fitbit.api.intraday";
69+
private static final String FITBIT_API_INTRADAY_ACCESS_DOC = "Set to true if the client has permissions to Fitbit Intraday API, false otherwise.";
70+
private static final boolean FITBIT_API_INTRADAY_ACCESS_DEFAULT = false;
71+
private static final String FITBIT_API_INTRADAY_ACCESS_DISPLAY = "Is Fitbit Intraday API available?";
72+
6773
public static final String FITBIT_USER_CREDENTIALS_DIR_CONFIG = "fitbit.user.dir";
6874
private static final String FITBIT_USER_CREDENTIALS_DIR_DOC = "Directory containing Fitbit user information and credentials. Only used if a file-based user repository is configured.";
6975
private static final String FITBIT_USER_CREDENTIALS_DIR_DISPLAY = "User directory";
@@ -95,21 +101,26 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
95101
private static final String FITBIT_SLEEP_CLASSIC_TOPIC_DISPLAY = "Classic sleep topic";
96102

97103
private static final String FITBIT_TIME_ZONE_TOPIC_CONFIG = "fitbit.time.zone.topic";
98-
private static final String FITBIT_TIME_ZONE_TOPIC_DOC = "Topic for Fitbit profile timezone";
104+
private static final String FITBIT_TIME_ZONE_TOPIC_DOC = "Topic for Fitbit profile time zone";
99105
private static final String FITBIT_TIME_ZONE_TOPIC_DEFAULT = "connect_fitbit_time_zone";
100-
private static final String FITBIT_TIME_ZONE_TOPIC_DISPLAY = "Timezone topic";
106+
private static final String FITBIT_TIME_ZONE_TOPIC_DISPLAY = "Time zone topic";
101107

102108
private static final String FITBIT_MAX_USERS_PER_POLL_CONFIG = "fitbit.max.users.per.poll";
103109
private static final String FITBIT_MAX_USERS_PER_POLL_DOC = "Maximum number of users to query in a single poll operation. Decrease this if memory constrains are pressing.";
104110
private static final int FITBIT_MAX_USERS_PER_POLL_DEFAULT = 100;
105111
private static final String FITBIT_MAX_USERS_PER_POLL_DISPLAY = "Maximum users per poll";
106112

113+
private static final String FITBIT_ACTIVITY_LOG_TOPIC_CONFIG = "fitbit.activity.log.topic";
114+
private static final String FITBIT_ACTIVITY_LOG_TOPIC_DOC = "Topic for Fitbit activity log.";
115+
private static final String FITBIT_ACTIVITY_LOG_TOPIC_DEFAULT = "connect_fitbit_activity_log";
116+
private static final String FITBIT_ACTIVITY_LOG_TOPIC_DISPLAY = "Activity log topic";
117+
107118
private final UserRepository userRepository;
108119
private final Headers clientCredentials;
109120

110121
@SuppressWarnings("unchecked")
111-
public FitbitRestSourceConnectorConfig(ConfigDef config, Map<String, String> parsedConfig) {
112-
super(config, parsedConfig);
122+
public FitbitRestSourceConnectorConfig(ConfigDef config, Map<String, String> parsedConfig, boolean doLog) {
123+
super(config, parsedConfig, doLog);
113124

114125
try {
115126
userRepository = ((Class<? extends UserRepository>)
@@ -125,8 +136,8 @@ public FitbitRestSourceConnectorConfig(ConfigDef config, Map<String, String> par
125136
this.clientCredentials = Headers.of("Authorization", "Basic " + credentialsBase64);
126137
}
127138

128-
public FitbitRestSourceConnectorConfig(Map<String, String> parsedConfig) {
129-
this(FitbitRestSourceConnectorConfig.conf(), parsedConfig);
139+
public FitbitRestSourceConnectorConfig(Map<String, String> parsedConfig, boolean doLog) {
140+
this(FitbitRestSourceConnectorConfig.conf(), parsedConfig, doLog);
130141
}
131142

132143
public static ConfigDef conf() {
@@ -173,6 +184,16 @@ public String toString() {
173184
Width.SHORT,
174185
FITBIT_API_SECRET_DISPLAY)
175186

187+
.define(FITBIT_API_INTRADAY_ACCESS_CONFIG,
188+
Type.BOOLEAN,
189+
true,
190+
Importance.MEDIUM,
191+
FITBIT_API_INTRADAY_ACCESS_DOC,
192+
group,
193+
++orderInGroup,
194+
Width.SHORT,
195+
FITBIT_API_INTRADAY_ACCESS_DISPLAY)
196+
176197
.define(FITBIT_USER_REPOSITORY_CONFIG,
177198
Type.CLASS,
178199
YamlUserRepository.class,
@@ -269,6 +290,17 @@ public String toString() {
269290
++orderInGroup,
270291
Width.SHORT,
271292
FITBIT_TIME_ZONE_TOPIC_DISPLAY)
293+
294+
.define(FITBIT_ACTIVITY_LOG_TOPIC_CONFIG,
295+
Type.STRING,
296+
FITBIT_ACTIVITY_LOG_TOPIC_DEFAULT,
297+
nonControlChar,
298+
Importance.LOW,
299+
FITBIT_ACTIVITY_LOG_TOPIC_DOC,
300+
group,
301+
++orderInGroup,
302+
Width.SHORT,
303+
FITBIT_ACTIVITY_LOG_TOPIC_DISPLAY)
272304
;
273305
}
274306

@@ -333,4 +365,12 @@ public Headers getClientCredentials() {
333365
public long getMaxUsersPerPoll() {
334366
return getInt(FITBIT_MAX_USERS_PER_POLL_CONFIG);
335367
}
368+
369+
public String getActivityLogTopic() {
370+
return getString(FITBIT_ACTIVITY_LOG_TOPIC_CONFIG);
371+
}
372+
373+
public boolean hasIntradayAccess() {
374+
return getBoolean(FITBIT_API_INTRADAY_ACCESS_CONFIG);
375+
}
336376
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,15 @@ public void start(Map<String, String> props) {
5252
executor.scheduleAtFixedRate(() -> {
5353
try {
5454
logger.info("Requesting latest user details...");
55-
Set<? extends User> newUsers = getConfig(props).getUserRepository().stream()
55+
Set<? extends User> newUsers = getConfig(props, false).getUserRepository().stream()
5656
.collect(Collectors.toSet());
5757
if (configuredUsers != null && !newUsers.equals(configuredUsers)) {
5858
logger.info("User info mismatch found. Requesting reconfiguration...");
5959
reconfigure();
6060
}
6161
} catch (IOException e) {
62-
e.printStackTrace();
62+
logger.warn("Failed to refresh users: {}", e.toString());
6363
}
64-
6564
},0, 5, TimeUnit.MINUTES);
6665
}
6766

@@ -73,9 +72,13 @@ public void stop() {
7372
configuredUsers = null;
7473
}
7574

75+
private FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf, boolean doLog) {
76+
return new FitbitRestSourceConnectorConfig(conf, doLog);
77+
}
78+
7679
@Override
7780
public FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf) {
78-
return new FitbitRestSourceConnectorConfig(conf);
81+
return getConfig(conf, true);
7982
}
8083

8184
@Override

0 commit comments

Comments
 (0)