Skip to content

Commit bba9625

Browse files
yatharthranjanyatharthranjan
authored
Merge pull request #55 from RADAR-base/firebase_user_repo
Firebase user repo
2 parents 1b7d783 + a26ca00 commit bba9625

15 files changed

+975
-54
lines changed

‎.editorconfig‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ insert_final_newline = true
1010
charset = utf-8
1111
indent_style = space
1212
indent_size = 2
13-
continuation_indent_size = 4
13+
ij_continuation_indent_size = 4
14+
max_line_length=100
1415

15-
[*.gradle,*.py]
16+
[{*.gradle, *.py}]
1617
indent_size = 4
17-
continuation_indent_size = 8
18+
ij_continuation_indent_size = 8

‎kafka-connect-fitbit-source/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ dependencies {
66

77
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion
88
implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion
9+
implementation 'com.google.firebase:firebase-admin:6.12.2'
910

1011
// Included in connector runtime
1112
compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java‎

Lines changed: 60 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.apache.kafka.common.config.ConfigDef;
3434
import org.apache.kafka.common.config.ConfigDef.Importance;
3535
import org.apache.kafka.common.config.ConfigDef.NonEmptyString;
36-
import org.apache.kafka.common.config.ConfigDef.Range;
3736
import org.apache.kafka.common.config.ConfigDef.Type;
3837
import org.apache.kafka.common.config.ConfigDef.Validator;
3938
import org.apache.kafka.common.config.ConfigDef.Width;
@@ -123,21 +122,22 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
123122
private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DISPLAY = "Intraday calories topic";
124123
private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DEFAULT = "connect_fitbit_intraday_calories";
125124

126-
private final UserRepository userRepository;
125+
public static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG = "fitbit.user.firebase.collection.fitbit.name";
126+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DOC = "Firestore Collection for retrieving Fitbit Auth details. Only used when a Firebase based user repository is used.";
127+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DISPLAY = "Firebase Fitbit collection name.";
128+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DEFAULT = "fitbit";
129+
130+
public static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG = "fitbit.user.firebase.collection.user.name";
131+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DOC = "Firestore Collection for retrieving User details. Only used when a Firebase based user repository is used.";
132+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DISPLAY = "Firebase User collection name.";
133+
private static final String FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DEFAULT = "users";
134+
135+
private UserRepository userRepository;
127136
private final Headers clientCredentials;
128137

129-
@SuppressWarnings("unchecked")
130138
public FitbitRestSourceConnectorConfig(ConfigDef config, Map<String, String> parsedConfig, boolean doLog) {
131139
super(config, parsedConfig, doLog);
132140

133-
try {
134-
userRepository = ((Class<? extends UserRepository>)
135-
getClass(FITBIT_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance();
136-
} catch (IllegalAccessException | InstantiationException
137-
| InvocationTargetException | NoSuchMethodException e) {
138-
throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e);
139-
}
140-
141141
String credentialString = getFitbitClient() + ":" + getFitbitClientSecret();
142142
String credentialsBase64 = Base64.getEncoder().encodeToString(
143143
credentialString.getBytes(StandardCharsets.UTF_8));
@@ -318,6 +318,26 @@ public String toString() {
318318
++orderInGroup,
319319
Width.SHORT,
320320
FITBIT_INTRADAY_CALORIES_TOPIC_DISPLAY)
321+
322+
.define(FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG,
323+
Type.STRING,
324+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DEFAULT,
325+
Importance.LOW,
326+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DOC,
327+
group,
328+
++orderInGroup,
329+
Width.SHORT,
330+
FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DISPLAY)
331+
332+
.define(FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG,
333+
Type.STRING,
334+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DEFAULT,
335+
Importance.LOW,
336+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DOC,
337+
group,
338+
++orderInGroup,
339+
Width.SHORT,
340+
FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_DISPLAY)
321341
;
322342
}
323343

@@ -333,11 +353,32 @@ public String getFitbitClientSecret() {
333353
return getPassword(FITBIT_API_SECRET_CONFIG).value();
334354
}
335355

356+
public UserRepository getUserRepository(UserRepository reuse) {
357+
if (reuse != null && reuse.getClass().equals(getClass(FITBIT_USER_REPOSITORY_CONFIG))) {
358+
userRepository = reuse;
359+
} else {
360+
userRepository = createUserRepository();
361+
}
362+
userRepository.initialize(this);
363+
return userRepository;
364+
}
365+
336366
public UserRepository getUserRepository() {
337367
userRepository.initialize(this);
338368
return userRepository;
339369
}
340370

371+
@SuppressWarnings("unchecked")
372+
public UserRepository createUserRepository() {
373+
try {
374+
return ((Class<? extends UserRepository>)
375+
getClass(FITBIT_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance();
376+
} catch (IllegalAccessException | InstantiationException
377+
| InvocationTargetException | NoSuchMethodException e) {
378+
throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e);
379+
}
380+
}
381+
341382
public String getFitbitIntradayStepsTopic() {
342383
return getString(FITBIT_INTRADAY_STEPS_TOPIC_CONFIG);
343384
}
@@ -398,4 +439,12 @@ public Duration getTooManyRequestsCooldownInterval() {
398439
public String getFitbitIntradayCaloriesTopic() {
399440
return getString(FITBIT_INTRADAY_CALORIES_TOPIC_CONFIG);
400441
}
442+
443+
public String getFitbitUserRepositoryFirestoreFitbitCollection() {
444+
return getString(FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG);
445+
}
446+
447+
public String getFitbitUserRepositoryFirestoreUserCollection() {
448+
return getString(FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG);
449+
}
401450
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java‎

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,11 @@
2828
import java.util.concurrent.ScheduledExecutorService;
2929
import java.util.concurrent.TimeUnit;
3030
import java.util.stream.Collectors;
31-
import java.util.stream.Stream;
32-
3331
import org.apache.kafka.common.config.ConfigDef;
3432
import org.apache.kafka.common.config.ConfigException;
3533
import org.radarbase.connect.rest.AbstractRestSourceConnector;
3634
import org.radarbase.connect.rest.fitbit.user.User;
35+
import org.radarbase.connect.rest.fitbit.user.UserRepository;
3736
import org.slf4j.Logger;
3837
import org.slf4j.LoggerFactory;
3938

@@ -42,26 +41,32 @@ public class FitbitSourceConnector extends AbstractRestSourceConnector {
4241
private static final Logger logger = LoggerFactory.getLogger(FitbitSourceConnector.class);
4342
private ScheduledExecutorService executor;
4443
private Set<? extends User> configuredUsers;
45-
44+
private UserRepository repository;
4645

4746
@Override
4847
public void start(Map<String, String> props) {
4948
super.start(props);
5049
executor = Executors.newSingleThreadScheduledExecutor();
5150

5251
executor.scheduleAtFixedRate(() -> {
53-
try {
54-
logger.info("Requesting latest user details...");
55-
Set<? extends User> newUsers = getConfig(props, false).getUserRepository().stream()
56-
.collect(Collectors.toSet());
57-
if (configuredUsers != null && !newUsers.equals(configuredUsers)) {
58-
logger.info("User info mismatch found. Requesting reconfiguration...");
59-
reconfigure();
52+
if (repository.hasPendingUpdates()) {
53+
try {
54+
logger.info("Requesting latest user details...");
55+
repository.applyPendingUpdates();
56+
Set<? extends User> newUsers =
57+
getConfig(props, false).getUserRepository(repository).stream()
58+
.collect(Collectors.toSet());
59+
if (configuredUsers != null && !newUsers.equals(configuredUsers)) {
60+
logger.info("User info mismatch found. Requesting reconfiguration...");
61+
reconfigure();
62+
}
63+
} catch (IOException e) {
64+
logger.warn("Failed to refresh users: {}", e.toString());
6065
}
61-
} catch (IOException e) {
62-
logger.warn("Failed to refresh users: {}", e.toString());
66+
} else {
67+
logger.info("No pending updates found. Not attempting to refresh users.");
6368
}
64-
},0, 5, TimeUnit.MINUTES);
69+
}, 0, 5, TimeUnit.MINUTES);
6570
}
6671

6772
@Override
@@ -78,7 +83,9 @@ private FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf, bool
7883

7984
@Override
8085
public FitbitRestSourceConnectorConfig getConfig(Map<String, String> conf) {
81-
return getConfig(conf, true);
86+
FitbitRestSourceConnectorConfig connectorConfig = getConfig(conf, true);
87+
repository = connectorConfig.getUserRepository(repository);
88+
return connectorConfig;
8289
}
8390

8491
@Override
@@ -94,14 +101,16 @@ public List<Map<String, String>> taskConfigs(int maxTasks) {
94101
private List<Map<String, String>> configureTasks(int maxTasks) {
95102
Map<String, String> baseConfig = config.originalsStrings();
96103
FitbitRestSourceConnectorConfig fitbitConfig = getConfig(baseConfig);
104+
if (repository == null) {
105+
repository = fitbitConfig.getUserRepository(null);
106+
}
97107
// Divide the users over tasks
98108
try {
99-
100-
List<Map<String, String>> userTasks = fitbitConfig.getUserRepository().stream()
109+
List<Map<String, String>> userTasks = fitbitConfig.getUserRepository(repository).stream()
101110
.map(User::getVersionedId)
102-
// group users based on their hashCode
103-
// in principle this allows for more efficient reconfigurations for a fixed number of tasks,
104-
// since that allows existing tasks to only handle small modifications users to handle.
111+
// group users based on their hashCode, in principle, this allows for more efficient
112+
// reconfigurations for a fixed number of tasks, since that allows existing tasks to
113+
// only handle small modifications users to handle.
105114
.collect(Collectors.groupingBy(
106115
u -> Math.abs(u.hashCode()) % maxTasks,
107116
Collectors.joining(",")))
@@ -114,7 +123,7 @@ private List<Map<String, String>> configureTasks(int maxTasks) {
114123
.collect(Collectors.toList());
115124
this.configuredUsers = fitbitConfig.getUserRepository().stream()
116125
.collect(Collectors.toSet());
117-
logger.info("Received userTask Configs {}" , userTasks);
126+
logger.info("Received userTask Configs {}", userTasks);
118127
return userTasks;
119128
} catch (IOException ex) {
120129
throw new ConfigException("Cannot read users", ex);

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/TokenAuthenticator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public Request authenticate(Route requestRoute, Response response) throws IOExce
5555
.header("Authorization", "Bearer " + newAccessToken)
5656
.build();
5757
} catch (NotAuthorizedException ex) {
58-
logger.error("Cannot get a new refresh token for user {}. Cancelling request.", user);
58+
logger.error("Cannot get a new refresh token for user {}. Cancelling request.", user, ex);
5959
return null;
6060
}
6161
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -85,23 +85,7 @@ public void initialize(RestSourceConnectorConfig config) {
8585
}
8686

8787
@Override
88-
public Stream<? extends User> stream() throws IOException {
89-
Instant nextFetchTime = nextFetch.get();
90-
Instant now = Instant.now();
91-
if (!now.isAfter(nextFetchTime)
92-
|| !nextFetch.compareAndSet(nextFetchTime, now.plus(FETCH_THRESHOLD))) {
93-
logger.debug("Providing cached user information...");
94-
return timedCachedUsers.stream();
95-
}
96-
97-
logger.info("Requesting user information from webservice");
98-
Request request = requestFor("users?source-type=FitBit").build();
99-
this.timedCachedUsers =
100-
this.<Users>makeRequest(request, USER_LIST_READER).getUsers().stream()
101-
.filter(u -> u.isComplete()
102-
&& (containedUsers.isEmpty() || containedUsers.contains(u.getVersionedId())))
103-
.collect(Collectors.toSet());
104-
88+
public Stream<? extends User> stream() {
10589
return this.timedCachedUsers.stream();
10690
}
10791

@@ -124,6 +108,28 @@ public String refreshAccessToken(User user) throws IOException, NotAuthorizedExc
124108
return credentials.getAccessToken();
125109
}
126110

111+
@Override
112+
public boolean hasPendingUpdates() {
113+
Instant nextFetchTime = nextFetch.get();
114+
Instant now = Instant.now();
115+
return now.isAfter(nextFetchTime);
116+
}
117+
118+
@Override
119+
public void applyPendingUpdates() throws IOException {
120+
logger.info("Requesting user information from webservice");
121+
Request request = requestFor("users?source-type=FitBit").build();
122+
this.timedCachedUsers =
123+
this.<Users>makeRequest(request, USER_LIST_READER).getUsers().stream()
124+
.filter(
125+
u ->
126+
u.isComplete()
127+
&& (containedUsers.isEmpty()
128+
|| containedUsers.contains(u.getVersionedId())))
129+
.collect(Collectors.toSet());
130+
nextFetch.set(Instant.now().plus(FETCH_THRESHOLD));
131+
}
132+
127133
private Request.Builder requestFor(String relativeUrl) {
128134
HttpUrl url = baseUrl.resolve(relativeUrl);
129135
if (url == null) {

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/UserRepository.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,19 @@ public interface UserRepository extends RestSourceTool {
5858
* @throws java.util.NoSuchElementException if the user does not exists in this repository.
5959
*/
6060
String refreshAccessToken(User user) throws IOException, NotAuthorizedException;
61+
62+
/**
63+
* The functions allows the repository to supply when there are pending updates.
64+
* This gives more control to the user repository in updating and caching users.
65+
* @return {@code true} if there are new updates available, {@code false} otherwise.
66+
*/
67+
boolean hasPendingUpdates();
68+
69+
/**
70+
* Apply any pending updates to users. This could include, for instance, refreshing a cache
71+
* of users with latest information.
72+
* This is called when {@link #hasPendingUpdates()} is {@code true}.
73+
* @throws IOException if there was an error when applying updates.
74+
*/
75+
void applyPendingUpdates() throws IOException;
6176
}

‎kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/YamlUserRepository.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,10 @@ private void updateUsers() {
112112
|| !nextFetch.compareAndSet(nextFetchTime, now.plus(FETCH_THRESHOLD))) {
113113
return;
114114
}
115+
forceUpdateUsers();
116+
}
115117

118+
private void forceUpdateUsers() {
116119
try {
117120
Map<String, LockedUser> newMap = Files.walk(credentialsDir)
118121
.filter(p -> Files.isRegularFile(p)
@@ -129,8 +132,6 @@ private void updateUsers() {
129132

130133
@Override
131134
public Stream<LocalUser> stream() {
132-
updateUsers();
133-
134135
Stream<LockedUser> users = this.users.values().stream()
135136
.filter(lockedTest(u -> u.getOAuth2Credentials().hasRefreshToken()));
136137
if (!configuredUsers.isEmpty()) {
@@ -177,6 +178,19 @@ public String refreshAccessToken(User user) throws IOException {
177178
return refreshAccessToken(user, NUM_RETRIES);
178179
}
179180

181+
@Override
182+
public boolean hasPendingUpdates() {
183+
Instant nextFetchTime = nextFetch.get();
184+
Instant now = Instant.now();
185+
return now.isAfter(nextFetchTime);
186+
}
187+
188+
@Override
189+
public void applyPendingUpdates() {
190+
forceUpdateUsers();
191+
nextFetch.set(Instant.now().plus(FETCH_THRESHOLD));
192+
}
193+
180194
/**
181195
* Refreshes the Fitbit access token on the current host, using the locally stored refresh token.
182196
* If successful, the tokens are locally stored.

0 commit comments

Comments
 (0)