Skip to content

Commit 53f5e8c

Browse files
authored
Merge pull request #99 from RADAR-base/update-dependencies
Update dependencies
2 parents 9b0809e + 21d43dd commit 53f5e8c

File tree

21 files changed

+246
-208
lines changed

21 files changed

+246
-208
lines changed

.editorconfig

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,5 @@ indent_size = 2
1313
ij_continuation_indent_size = 4
1414
max_line_length=100
1515

16-
[{*.gradle, *.py}]
16+
[*.{gradle,py,kt,kts}]
1717
indent_size = 4
18-
ij_continuation_indent_size = 8

Dockerfile

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,18 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
FROM --platform=$BUILDPLATFORM gradle:7.5-jdk11 as builder
15+
FROM --platform=$BUILDPLATFORM gradle:8.1-jdk11 as builder
1616

1717
RUN mkdir /code
1818
WORKDIR /code
1919

2020
ENV GRADLE_USER_HOME=/code/.gradlecache \
2121
GRADLE_OPTS="-Dorg.gradle.vfs.watch=false -Djdk.lang.Process.launchMechanism=vfork"
2222

23-
COPY ./build.gradle ./settings.gradle ./gradle.properties /code/
24-
COPY kafka-connect-rest-source/build.gradle /code/kafka-connect-rest-source/
25-
COPY kafka-connect-fitbit-source/build.gradle /code/kafka-connect-fitbit-source/
23+
COPY buildSrc /code/buildSrc
24+
COPY ./build.gradle.kts ./settings.gradle.kts ./gradle.properties /code/
25+
COPY kafka-connect-rest-source/build.gradle.kts /code/kafka-connect-rest-source/
26+
COPY kafka-connect-fitbit-source/build.gradle.kts /code/kafka-connect-fitbit-source/
2627

2728
RUN gradle downloadDependencies copyDependencies
2829

@@ -31,13 +32,14 @@ COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src
3132

3233
RUN gradle jar
3334

34-
FROM confluentinc/cp-kafka-connect-base:7.2.2
35+
FROM confluentinc/cp-kafka-connect-base:7.4.0
3536

3637
MAINTAINER Joris Borgdorff <[email protected]>
3738

3839
LABEL description="Kafka REST API Source connector"
3940

40-
ENV CONNECT_PLUGIN_PATH=/usr/share/java/kafka-connect/plugins
41+
ENV CONNECT_PLUGIN_PATH="/usr/share/java/kafka-connect/plugins" \
42+
WAIT_FOR_KAFKA="1"
4143

4244
# To isolate the classpath from the plugin path as recommended
4345
COPY --from=builder /code/kafka-connect-rest-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-rest-source/

build.gradle

Lines changed: 0 additions & 62 deletions
This file was deleted.

build.gradle.kts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import org.radarbase.gradle.plugin.radarKotlin
2+
3+
plugins {
4+
id("org.radarbase.radar-root-project") version Versions.radarCommons
5+
id("org.radarbase.radar-dependency-management") version Versions.radarCommons
6+
id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false
7+
}
8+
9+
description = "Kafka connector for REST API sources"
10+
11+
radarRootProject {
12+
projectVersion.set(Versions.project)
13+
}
14+
15+
subprojects {
16+
apply(plugin = "org.radarbase.radar-kotlin")
17+
18+
radarKotlin {
19+
javaVersion.set(Versions.java)
20+
kotlinVersion.set(Versions.kotlin)
21+
slf4jVersion.set(Versions.slf4j)
22+
log4j2Version.set(Versions.log4j2)
23+
junitVersion.set(Versions.junit)
24+
}
25+
}

buildSrc/build.gradle.kts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
plugins {
2+
kotlin("jvm") version "1.8.10"
3+
}
4+
5+
repositories {
6+
mavenCentral()
7+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
object Versions {
2+
const val project = "0.4.1-SNAPSHOT"
3+
4+
const val java = 11
5+
const val kotlin = "1.8.21"
6+
7+
const val radarCommons = "1.0.0"
8+
const val confluent = "7.4.0"
9+
const val kafka = "$confluent-ce"
10+
11+
// From image
12+
const val jackson = "2.14.2"
13+
14+
const val log4j2 = "2.20.0"
15+
const val slf4j = "2.0.7"
16+
17+
const val okhttp = "4.11.0"
18+
19+
const val managementPortal = "2.0.0"
20+
21+
const val firebaseAdmin = "9.1.0"
22+
const val radarSchemas = "0.8.3"
23+
24+
const val junit = "5.9.3"
25+
const val hamcrest = "2.2"
26+
const val wiremock = "2.27.2"
27+
const val mockito = "5.3.1"
28+
}

docker/kafka-wait

Lines changed: 74 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,87 @@
11
#!/bin/bash
22

3-
# Check if variables exist
4-
if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then
5-
echo "CONNECT_ZOOKEEPER_CONNECT is not defined"
6-
exit 2
3+
if [ "$WAIT_FOR_KAFKA" != "1" ]; then
4+
echo "Starting without checking for Kafka availability"
5+
exit 0
76
fi
87

9-
if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then
10-
echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined"
11-
exit 4
8+
max_timeout=32
9+
10+
11+
IS_TEMP=0
12+
13+
if [ -z "$COMMAND_CONFIG_FILE_PATH" ]; then
14+
COMMAND_CONFIG_FILE_PATH="$(mktemp)"
15+
IS_TEMP=1
16+
fi
17+
18+
if [ ! -f "$COMMAND_CONFIG_FILE_PATH" ] || [ $IS_TEMP = 1 ]; then
19+
while IFS='=' read -r -d '' n v; do
20+
if [[ "$n" == "CONNECT_"* ]]; then
21+
name="${n/CONNECT_/""}" # remove first "CONNECT_"
22+
name="${name,,}" # lower case
23+
name="${name//_/"."}" # replace all '_' with '.'
24+
echo "$name=$v" >> ${COMMAND_CONFIG_FILE_PATH}
25+
fi
26+
done < <(env -0)
1227
fi
1328

14-
KAFKA_BROKERS=${KAFKA_BROKERS:-3}
29+
# Check if variables exist
30+
if [ -z "$CONNECT_BOOTSTRAP_SERVERS" ]; then
31+
echo "CONNECT_BOOTSTRAP_SERVERS is not defined"
32+
else
33+
KAFKA_BROKERS=${KAFKA_BROKERS:-3}
1534

16-
max_timeout=32
35+
tries=10
36+
timeout=1
37+
while true; do
38+
KAFKA_CHECK=$(kafka-broker-api-versions --bootstrap-server "$CONNECT_BOOTSTRAP_SERVERS" --command-config "${COMMAND_CONFIG_FILE_PATH}" | grep "(id: " | wc -l)
1739

18-
tries=10
19-
timeout=1
20-
while true; do
21-
ZOOKEEPER_CHECK=$(zookeeper-shell ${CONNECT_ZOOKEEPER_CONNECT} <<< "ls /brokers/ids" | tail -n 1)
22-
echo "Zookeeper response: ${ZOOKEEPER_CHECK}"
23-
ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')"
40+
if [ "$KAFKA_CHECK" -ge "$KAFKA_BROKERS" ]; then
41+
echo "Kafka brokers available."
42+
break
43+
fi
2444

25-
IFS=',' read -r -a array <<< "${ZOOKEEPER_CHECK}"
26-
LENGTH=${#array[@]}
27-
if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then
28-
echo "Kafka brokers available."
29-
break
30-
fi
45+
tries=$((tries - 1))
46+
if [ ${tries} -eq 0 ]; then
47+
echo "FAILED: KAFKA BROKERs NOT READY."
48+
exit 5
49+
fi
50+
echo "Expected $KAFKA_BROKERS brokers but found only $KAFKA_CHECK. Waiting $timeout second before retrying ..."
51+
sleep ${timeout}
52+
if [ ${timeout} -lt ${max_timeout} ]; then
53+
timeout=$((timeout * 2))
54+
fi
55+
done
3156

32-
tries=$((tries - 1))
33-
if [ ${tries} -eq 0 ]; then
34-
echo "FAILED: KAFKA BROKERs NOT READY."
35-
exit 5
36-
fi
37-
echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..."
38-
sleep ${timeout}
39-
if [ ${timeout} -lt ${max_timeout} ]; then
40-
timeout=$((timeout * 2))
41-
fi
42-
done
57+
echo "Kafka is available."
58+
fi
4359

44-
tries=10
45-
timeout=1
46-
while true; do
47-
if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then
48-
echo "Schema registry available."
49-
break
50-
fi
51-
tries=$((tries - 1))
52-
if [ $tries -eq 0 ]; then
53-
echo "FAILED TO REACH SCHEMA REGISTRY."
54-
exit 6
55-
fi
56-
echo "Failed to reach schema registry. Retrying in ${timeout} seconds."
57-
sleep ${timeout}
58-
if [ ${timeout} -lt ${max_timeout} ]; then
59-
timeout=$((timeout * 2))
60-
fi
61-
done
60+
if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then
61+
echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined"
62+
else
63+
tries=10
64+
timeout=1
65+
while true; do
66+
if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then
67+
echo "Schema registry available."
68+
break
69+
fi
70+
tries=$((tries - 1))
71+
if [ $tries -eq 0 ]; then
72+
echo "FAILED TO REACH SCHEMA REGISTRY."
73+
exit 6
74+
fi
75+
echo "Failed to reach schema registry. Retrying in ${timeout} seconds."
76+
sleep ${timeout}
77+
if [ ${timeout} -lt ${max_timeout} ]; then
78+
timeout=$((timeout * 2))
79+
fi
80+
done
6281

82+
echo "Schema registry is available."
83+
fi
6384

64-
echo "Kafka is available. Ready to go!"
85+
if [ $IS_TEMP = 1 ]; then
86+
/bin/rm -f "$COMMAND_CONFIG_FILE_PATH"
87+
fi

docker/launch

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,20 +33,27 @@ if [ "$KAFKA_JMX_PORT" ]; then
3333
export JMX_PORT=$KAFKA_JMX_PORT
3434
export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT"
3535
fi
36-
37-
# Busy waiting loop that waits untill all topic are available
36+
#
37+
## Busy waiting loop that waits until all topic are available
3838
echo "===> Wait for infrastructure ..."
3939
kafka-wait
4040
radar_check=$?
4141
if [ "$radar_check" -ne 0 ]; then
42-
exit ${radar_check}
42+
exit $radar_check
4343
fi
4444

4545
echo "===> Launching ${COMPONENT} ..."
4646
# Add our jar to the classpath so that the custom classes can be loaded first.
4747
# And this also makes sure that the CLASSPATH does not start with ":/etc/..."
4848
# other jars are loaded via the plugin path
49-
#export CLASSPATH="/etc/${COMPONENT}/kafka-connect-mongodb-sink/*"
49+
if [ -z "$CLASSPATH" ]; then
50+
export CLASSPATH="/etc/kafka-connect/jars/*"
51+
fi
5052

51-
# execute connector in standalone mode
52-
exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties /etc/"${COMPONENT}"/"${CONNECTOR_PROPERTY_FILE_PREFIX}"*.properties
53+
if [ -z "$CONNECTOR_PROPERTY_FILE_PREFIX" ]; then
54+
# execute connector in distributed mode
55+
exec connect-distributed /etc/"${COMPONENT}"/"${COMPONENT}".properties
56+
else
57+
# execute connector in standalone mode
58+
exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties /etc/"${COMPONENT}"/"${CONNECTOR_PROPERTY_FILE_PREFIX}"*.properties
59+
fi

gradle.properties

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m
2+
3+
kotlin.code.style=official
4+
5+
public.gpr.user=radar-public
6+
public.gpr.token=Z2hwX0h0d0FHSmJzeEpjenBlUVIycVhWb0RpNGdZdHZnZzJTMFVJZA==

gradle/wrapper/gradle-wrapper.jar

1.29 KB
Binary file not shown.

0 commit comments

Comments
 (0)