Skip to content

Commit 743ebeb

Browse files
yatharthranjanyatharthranjan
authored
Merge pull request #57 from RADAR-base/release-0.3.0
Release 0.3.0
2 parents 98c260e + 6bafb92 commit 743ebeb

20 files changed

+1016
-74
lines changed

‎.editorconfig‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ insert_final_newline = true
1010
charset = utf-8
1111
indent_style = space
1212
indent_size = 2
13-
continuation_indent_size = 4
13+
ij_continuation_indent_size = 4
14+
max_line_length=100
1415

15-
[*.gradle,*.py]
16+
[{*.gradle, *.py}]
1617
indent_size = 4
17-
continuation_indent_size = 8
18+
ij_continuation_indent_size = 8

‎build.gradle‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,17 @@ subprojects {
1111
apply plugin: 'java-library'
1212

1313
group = 'org.radarbase'
14-
version = '0.2.4'
14+
version = '0.3.0'
1515

1616
sourceCompatibility = 1.8
1717
targetCompatibility = 1.8
1818

1919
repositories {
2020
mavenCentral()
21-
maven { url "http://packages.confluent.io/maven/" }
22-
maven { url "http://repo.maven.apache.org/maven2" }
21+
maven { url "https://packages.confluent.io/maven/" }
22+
maven { url "https://repo.maven.apache.org/maven2" }
2323
jcenter()
24-
maven { url 'http://oss.jfrog.org/artifactory/oss-snapshot-local/' }
24+
maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local/' }
2525
}
2626
}
2727

‎kafka-connect-fitbit-source/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ dependencies {
66

77
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion
88
implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion
9+
implementation 'com.google.firebase:firebase-admin:6.12.2'
910

1011
// Included in connector runtime
1112
compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java‎

Lines changed: 60 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.apache.kafka.common.config.ConfigDef;
3434
import org.apache.kafka.common.config.ConfigDef.Importance;
3535
import org.apache.kafka.common.config.ConfigDef.NonEmptyString;
36-
import org.apache.kafka.common.config.ConfigDef.Range;
3736
import org.apache.kafka.common.config.ConfigDef.Type;
3837
import org.apache.kafka.common.config.ConfigDef.Validator;
3938
import org.apache.kafka.common.config.ConfigDef.Width;
@@ -123,21 +122,22 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
123122
private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DISPLAY = "Intraday calories topic";
124123
private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DEFAULT = "connect_fitbit_intraday_calories";
125124

126-
private final UserRepository userRepository;
125+
public static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG = "fitbit.user.firebase.collection.fitbit.name";
126+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DOC = "Firestore Collection for retrieving Fitbit Auth details. Only used when a Firebase based user repository is used.";
127+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DISPLAY = "Firebase Fitbit collection name.";
128+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DEFAULT = "fitbit";
129+
130+
public static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG = "fitbit.user.firebase.collection.user.name";
131+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DOC = "Firestore Collection for retrieving User details. Only used when a Firebase based user repository is used.";
132+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DISPLAY = "Firebase User collection name.";
133+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DEFAULT = "users";
134+
135+
private UserRepository userRepository;
127136
private final Headers clientCredentials;
128137

129-
@SuppressWarnings("unchecked")
130138
public FitbitRestSourceConnectorConfig(ConfigDef config, Map<String, String> parsedConfig, boolean doLog) {
131139
super(config, parsedConfig, doLog);
132140

133-
try {
134-
userRepository = ((Class<? extends UserRepository>)
135-
getClass(FITBIT_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance();
136-
} catch (IllegalAccessException | InstantiationException
137-
| InvocationTargetException | NoSuchMethodException e) {
138-
throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e);
139-
}
140-
141141
String credentialString = getFitbitClient() + ":" + getFitbitClientSecret();
142142
String credentialsBase64 = Base64.getEncoder().encodeToString(
143143
credentialString.getBytes(StandardCharsets.UTF_8));
@@ -318,6 +318,26 @@ public String toString() {
318318
++orderInGroup,
319319
Width.SHORT,
320320
FITBIT_INTRADAY_CALORIES_TOPIC_DISPLAY)
321+
322+
.define(FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG,
323+
Type.STRING,
324+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DEFAULT,
325+
Importance.LOW,
326+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DOC,
327+
group,
328+
++orderInGroup,
329+
Width.SHORT,
330+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DISPLAY)
331+
332+
.define(FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG,
333+
Type.STRING,
334+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DEFAULT,
335+
Importance.LOW,
336+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DOC,
337+
group,
338+
++orderInGroup,
339+
Width.SHORT,
340+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DISPLAY)
321341
;
322342
}
323343

@@ -333,11 +353,32 @@ public String getFitbitClientSecret() {
333353
return getPassword(FITBIT_API_SECRET_CONFIG).value();
334354
}
335355

356+
public UserRepository getUserRepository(UserRepository reuse) {
357+
if (reuse != null && reuse.getClass().equals(getClass(FITBIT_USER_REPOSITORY_CONFIG))) {
358+
userRepository = reuse;
359+
} else {
360+
userRepository = createUserRepository();
361+
}
362+
userRepository.initialize(this);
363+
return userRepository;
364+
}
365+
336366
public UserRepository getUserRepository() {
337367
userRepository.initialize(this);
338368
return userRepository;
339369
}
340370

371+
@SuppressWarnings("unchecked")
372+
public UserRepository createUserRepository() {
373+
try {
374+
return ((Class<? extends UserRepository>)
375+
getClass(FITBIT_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance();
376+
} catch (IllegalAccessException | InstantiationException
377+
| InvocationTargetException | NoSuchMethodException e) {
378+
throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e);
379+
}
380+
}
381+
341382
public String getFitbitIntradayStepsTopic() {
342383
return getString(FITBIT_INTRADAY_STEPS_TOPIC_CONFIG);
343384
}
@@ -398,4 +439,12 @@ public Duration getTooManyRequestsCooldownInterval() {
398439
public String getFitbitIntradayCaloriesTopic() {
399440
return getString(FITBIT_INTRADAY_CALORIES_TOPIC_CONFIG);
400441
}
442+
443+
public String getFitbitUserRepositoryFirestoreFitbitCollection() {
444+
return getString(FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG);
445+
}
446+
447+
public String getFitbitUserRepositoryFirestoreUserCollection() {
448+
return getString(FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG);
449+
}
401450
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java‎

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,11 @@
2828
import java.util.concurrent.ScheduledExecutorService;
2929
import java.util.concurrent.TimeUnit;
3030
import java.util.stream.Collectors;
31-
import java.util.stream.Stream;
32-
3331
import org.apache.kafka.common.config.ConfigDef;
3432
import org.apache.kafka.common.config.ConfigException;
3533
import org.radarbase.connect.rest.AbstractRestSourceConnector;
3634
import org.radarbase.connect.rest.fitbit.user.User;
35+
import org.radarbase.connect.rest.fitbit.user.UserRepository;
3736
import org.slf4j.Logger;
3837
import org.slf4j.LoggerFactory;
3938

@@ -42,26 +41,32 @@ public class FitbitSourceConnector extends AbstractRestSourceConnector {
4241
private static final Logger logger = LoggerFactory.getLogger(FitbitSourceConnector.class);
4342
private ScheduledExecutorService executor;
4443
private Set<? extends User> configuredUsers;
45-
44+
private UserRepository repository;
4645

4746
@Override
4847
public void start(Map<String, String> props) {
4948
super.start(props);
5049
executor = Executors.newSingleThreadScheduledExecutor();
5150

5251
executor.scheduleAtFixedRate(() -> {
53-
try {
54-
logger.info("Requesting latest user details...");
55-
Set<? extends User> newUsers = getConfig(props, false).getUserRepository().stream()
56-
.collect(Collectors.toSet());
57-
if (configuredUsers != null && !newUsers.equals(configuredUsers)) {
58-
logger.info("User info mismatch found. Requesting reconfiguration...");
59-
reconfigure();
52+
if (repository.hasPendingUpdates()) {
53+
try {
54+
logger.info("Requesting latest user details...");
55+
repository.applyPendingUpdates();
56+
Set<? extends User> newUsers =
57+
getConfig(props, false).getUserRepository(repository).stream()
58+
.collect(Collectors.toSet());
59+
if (configuredUsers != null && !newUsers.equals(configuredUsers)) {
60+
logger.info("User info mismatch found. Requesting reconfiguration...");
61+
reconfigure();
62+
}
63+
} catch (IOException e) {
64+
logger.warn("Failed to refresh users: {}", e.toString());
6065
}
61-
} catch (IOException e) {
62-
logger.warn("Failed to refresh users: {}", e.toString());
66+
} else {
67+
logger.info("No pending updates found. Not attempting to refresh users.");
6368
}
64-
},0, 5, TimeUnit.MINUTES);
69+
}, 0, 5, TimeUnit.MINUTES);
6570
}
6671

6772
@Override
@@ -78,7 +83,9 @@ private FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf, bool
7883

7984
@Override
8085
public FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf) {
81-
return getConfig(conf, true);
86+
FitbitRestSourceConnectorConfig connectorConfig = getConfig(conf, true);
87+
repository = connectorConfig.getUserRepository(repository);
88+
return connectorConfig;
8289
}
8390

8491
@Override
@@ -94,14 +101,16 @@ public List<Map<String, String>> taskConfigs(int maxTasks) {
94101
private List<Map<String, String>> configureTasks(int maxTasks) {
95102
Map<String, String> baseConfig = config.originalsStrings();
96103
FitbitRestSourceConnectorConfig fitbitConfig = getConfig(baseConfig);
104+
if (repository == null) {
105+
repository = fitbitConfig.getUserRepository(null);
106+
}
97107
// Divide the users over tasks
98108
try {
99-
100-
List<Map<String, String>> userTasks = fitbitConfig.getUserRepository().stream()
101-
.map(User::getId)
102-
// group users based on their hashCode
103-
// in principle this allows for more efficient reconfigurations for a fixed number of tasks,
104-
// since that allows existing tasks to only handle small modifications users to handle.
109+
List<Map<String, String>> userTasks = fitbitConfig.getUserRepository(repository).stream()
110+
.map(User::getVersionedId)
111+
// group users based on their hashCode, in principle, this allows for more efficient
112+
// reconfigurations for a fixed number of tasks, since that allows existing tasks to
113+
// only handle small modifications users to handle.
105114
.collect(Collectors.groupingBy(
106115
u -> Math.abs(u.hashCode()) % maxTasks,
107116
Collectors.joining(",")))
@@ -114,7 +123,7 @@ private List<Map<String, String>> configureTasks(int maxTasks) {
114123
.collect(Collectors.toList());
115124
this.configuredUsers = fitbitConfig.getUserRepository().stream()
116125
.collect(Collectors.toSet());
117-
logger.info("Received userTask Configs {}" , userTasks);
126+
logger.info("Received userTask Configs {}", userTasks);
118127
return userTasks;
119128
} catch (IOException ex) {
120129
throw new ConfigException("Cannot read users", ex);

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public OkHttpClient getClient(User user) {
105105
public Map<String, Map<String, Object>> getPartitions(String route) {
106106
try {
107107
return userRepository.stream()
108-
.collect(Collectors.toMap(User::getId, u -> getPartition(route, u)));
108+
.collect(Collectors.toMap(User::getVersionedId, u -> getPartition(route, u)));
109109
} catch (IOException e) {
110110
logger.warn("Failed to initialize user partitions for route {}: {}", route, e.toString());
111111
return Collections.emptyMap();
@@ -114,7 +114,7 @@ public Map<String, Map<String, Object>> getPartitions(String route) {
114114

115115
public Map<String, Object> getPartition(String route, User user) {
116116
Map<String, Object> partition = new HashMap<>(4);
117-
partition.put("user", user.getId());
117+
partition.put("user", user.getVersionedId());
118118
partition.put("route", route);
119119
return partition;
120120
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/TokenAuthenticator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public Request authenticate(Route requestRoute, Response response) throws IOExce
5555
.header("Authorization", "Bearer " + newAccessToken)
5656
.build();
5757
} catch (NotAuthorizedException ex) {
58-
logger.error("Cannot get a new refresh token for user {}. Cancelling request.", user);
58+
logger.error("Cannot get a new refresh token for user {}. Cancelling request.", user, ex);
5959
return null;
6060
}
6161
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ public void initialize(RestSourceConnectorConfig config) {
143143
@Override
144144
public void requestSucceeded(RestRequest request, SourceRecord record) {
145145
lastPollPerUser.put(((FitbitRestRequest) request).getUser().getId(), lastPoll);
146-
String userKey = ((FitbitRestRequest) request).getUser().getId();
146+
String userKey = ((FitbitRestRequest) request).getUser().getVersionedId();
147147
Instant offset = Instant.ofEpochMilli((Long) record.sourceOffset().get(TIMESTAMP_OFFSET_KEY));
148148
offsets.put(userKey, offset);
149149
}
@@ -154,7 +154,7 @@ public void requestEmpty(RestRequest request) {
154154
FitbitRestRequest fitbitRequest = (FitbitRestRequest) request;
155155
Instant endOffset = fitbitRequest.getDateRange().end().toInstant();
156156
if (DAYS.between(endOffset, lastPoll) >= HISTORICAL_TIME_DAYS) {
157-
String key = fitbitRequest.getUser().getId();
157+
String key = fitbitRequest.getUser().getVersionedId();
158158
offsets.put(key, endOffset);
159159
}
160160
}
@@ -197,7 +197,7 @@ public Stream<FitbitRestRequest> requests() {
197197
return userRepository.stream()
198198
.map(u -> new AbstractMap.SimpleImmutableEntry<>(u, nextPoll(u)))
199199
.filter(u -> lastPoll.isAfter(u.getValue()))
200-
.sorted(Comparator.comparing(Map.Entry::getValue))
200+
.sorted(Map.Entry.comparingByValue())
201201
.flatMap(u -> this.createRequests(u.getKey()))
202202
.filter(Objects::nonNull);
203203
} catch (IOException e) {
@@ -216,7 +216,7 @@ public Instant getTimeOfNextRequest() {
216216
}
217217

218218
private Map<String, Object> getPartition(User user) {
219-
return partitions.computeIfAbsent(user.getId(),
219+
return partitions.computeIfAbsent(user.getVersionedId(),
220220
k -> generator.getPartition(routeName, user));
221221
}
222222

@@ -279,7 +279,7 @@ public Instant getLastPoll() {
279279
}
280280

281281
protected Instant getOffset(User user) {
282-
return offsets.getOrDefault(user.getId(), user.getStartDate().minus(ONE_NANO));
282+
return offsets.getOrDefault(user.getVersionedId(), user.getStartDate().minus(ONE_NANO));
283283
}
284284

285285
/**

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/LocalUser.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
public class LocalUser implements User {
3535
private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[^a-zA-Z0-9_-]");
3636
private String id;
37+
private String version;
3738
private String externalUserId;
3839
private String projectId;
3940
private String userId;
@@ -94,9 +95,19 @@ public void setFitbitUserId(String id) {
9495
this.externalUserId = id;
9596
}
9697

98+
@Override
99+
public String getVersion() {
100+
return version;
101+
}
102+
103+
public void setVersion(String version) {
104+
this.version = version;
105+
}
106+
97107
public LocalUser copy() {
98108
LocalUser copy = new LocalUser();
99109
copy.id = id;
110+
copy.version = version;
100111
copy.externalUserId = externalUserId;
101112
copy.projectId = projectId;
102113
copy.userId = userId;
@@ -116,7 +127,8 @@ public synchronized SchemaAndValue getObservationKey(AvroData avroData) {
116127

117128
@Override
118129
public String toString() {
119-
return "LocalUser{" + "id='" + id + '\''
130+
return "LocalUser{id='" + id + '\''
131+
+ ", version='" + version + '\''
120132
+ ", externalUserId='" + externalUserId + '\''
121133
+ ", projectId='" + projectId + '\''
122134
+ ", userId='" + userId + '\''
@@ -134,19 +146,17 @@ public boolean equals(Object o) {
134146
}
135147
LocalUser localUser = (LocalUser) o;
136148
return Objects.equals(id, localUser.id)
149+
&& Objects.equals(version, localUser.version)
137150
&& Objects.equals(externalUserId, localUser.externalUserId)
138151
&& Objects.equals(projectId, localUser.projectId)
139152
&& Objects.equals(userId, localUser.userId)
140153
&& Objects.equals(sourceId, localUser.sourceId)
141154
&& Objects.equals(startDate, localUser.startDate)
142-
&& Objects.equals(endDate, localUser.endDate)
143-
&& Objects.equals(observationKey, localUser.observationKey);
155+
&& Objects.equals(endDate, localUser.endDate);
144156
}
145157

146158
@Override
147159
public int hashCode() {
148-
149-
return Objects
150-
.hash(id, externalUserId, projectId, userId, sourceId, startDate, endDate, observationKey);
160+
return Objects.hash(id, version);
151161
}
152162
}

0 commit comments

Comments
 (0)