Skip to content

Commit f123db2

Browse files
mpgxviimpgxvii
authored
Merge pull request #113 from RADAR-base/backoff-time
Minor Oura connector fixes
2 parents 001ff4d + e60ddd0 commit f123db2

File tree

23 files changed

+924
-170
lines changed

23 files changed

+924
-170
lines changed

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ jobs:
104104
- name: Build docker
105105
uses: docker/build-push-action@v3
106106
with:
107+
file: ./kafka-connect-fitbit-source/Dockerfile
107108
context: .
108109
cache-from: type=local,src=/tmp/.buildx-cache
109110
cache-to: ${{ steps.cache-parameters.outputs.cache-to }}

‎.github/workflows/oura.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Continuous integration, including test and integration test
2+
name: Main Oura test
3+
4+
# Run in master and dev branches and in all pull requests to those branches
5+
on:
6+
push:
7+
branches: [ master, dev ]
8+
pull_request:
9+
branches: [ master, dev ]
10+
11+
env:
12+
DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
13+
14+
jobs:
15+
# Build and test the code
16+
kotlin:
17+
# The type of runner that the job will run on
18+
runs-on: ubuntu-latest
19+
20+
# Steps represent a sequence of tasks that will be executed as part of the job
21+
steps:
22+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
23+
- uses: actions/checkout@v3
24+
25+
- uses: actions/setup-java@v3
26+
with:
27+
distribution: temurin
28+
java-version: 11
29+
30+
- name: Setup Gradle
31+
uses: gradle/gradle-build-action@v2
32+
33+
- name: Compile code
34+
run: gradle assemble
35+
working-directory: ./kafka-connect-oura-source
36+
37+
# Gradle check
38+
- name: Check
39+
run: gradle check
40+
working-directory: ./kafka-connect-oura-source
41+
42+
docker:
43+
# The type of runner that the job will run on
44+
runs-on: ubuntu-latest
45+
46+
# Steps represent a sequence of tasks that will be executed as part of the job
47+
steps:
48+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
49+
- uses: actions/checkout@v3
50+
51+
- name: Docker build parameters
52+
id: docker_params
53+
run: |
54+
HAS_DOCKER_LOGIN=${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
55+
echo "has_docker_login=$HAS_DOCKER_LOGIN" >> $GITHUB_OUTPUT
56+
if [ "${{ github.event_name == 'pull_request' }}" = "true" ] || [ "$HAS_DOCKER_LOGIN" = "false" ]; then
57+
echo "push=false" >> $GITHUB_OUTPUT
58+
echo "load=true" >> $GITHUB_OUTPUT
59+
echo "platforms=linux/amd64" >> $GITHUB_OUTPUT
60+
else
61+
echo "push=true" >> $GITHUB_OUTPUT
62+
echo "load=false" >> $GITHUB_OUTPUT
63+
echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT
64+
fi
65+
66+
- name: Cache Docker layers
67+
id: cache_buildx
68+
uses: actions/cache@v3
69+
with:
70+
path: /tmp/.buildx-cache
71+
key: ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-${{ hashFiles('**/Dockerfile', '**/*.gradle', 'gradle.properties', '.dockerignore', '*/src/main/**', 'docker/**') }}
72+
restore-keys: |
73+
${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-
74+
${{ runner.os }}-buildx-
75+
76+
- name: Login to Docker Hub
77+
if: steps.docker_params.outputs.has_docker_login == 'true'
78+
uses: docker/login-action@v2
79+
with:
80+
username: ${{ secrets.DOCKERHUB_USERNAME }}
81+
password: ${{ secrets.DOCKERHUB_TOKEN }}
82+
83+
# Add Docker labels and tags
84+
- name: Docker meta
85+
id: docker_meta
86+
uses: docker/metadata-action@v4
87+
with:
88+
images: ${{ env.DOCKER_IMAGE }}
89+
90+
# Setup docker build environment
91+
- name: Set up QEMU
92+
uses: docker/setup-qemu-action@v2
93+
94+
- name: Set up Docker Buildx
95+
uses: docker/setup-buildx-action@v2
96+
97+
- name: Cache parameters
98+
id: cache-parameters
99+
run: |
100+
if [ "${{ steps.cache_buildx.outputs.cache-hit }}" = "true" ]; then
101+
echo "::set-output name=cache-to::"
102+
else
103+
echo "::set-output name=cache-to::type=local,dest=/tmp/.buildx-cache-new,mode=max"
104+
fi
105+
106+
- name: Build docker
107+
uses: docker/build-push-action@v3
108+
with:
109+
file: ./kafka-connect-oura-source/Dockerfile
110+
context: .
111+
cache-from: type=local,src=/tmp/.buildx-cache
112+
cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
113+
platforms: ${{ steps.docker_params.outputs.platforms }}
114+
load: ${{ steps.docker_params.outputs.load }}
115+
push: ${{ steps.docker_params.outputs.push }}
116+
tags: ${{ steps.docker_meta.outputs.tags }}
117+
# Use runtime labels from docker_meta as well as fixed labels
118+
labels: |
119+
${{ steps.docker_meta.outputs.labels }}
120+
maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
121+
org.opencontainers.image.description=RADAR-base upload connector backend application
122+
org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
123+
org.opencontainers.image.vendor=RADAR-base
124+
org.opencontainers.image.licenses=Apache-2.0
125+
126+
- name: Pull docker image
127+
if: steps.docker_params.outputs.load == 'false'
128+
run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
129+
130+
- name: Inspect docker image
131+
run: |
132+
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
133+
docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} curl --version
134+
135+
136+
# https://github.com/docker/build-push-action/issues/252
137+
# https://github.com/moby/buildkit/issues/1896
138+
- name: Move docker build cache
139+
if: steps.cache_buildx.outputs.cache-hit != 'true'
140+
run: |
141+
rm -rf /tmp/.buildx-cache
142+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/release.yml renamed to .github/workflows/release-fitbit.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ jobs:
7979
- name: Build docker
8080
uses: docker/build-push-action@v3
8181
with:
82+
file: ./kafka-connect-fitbit-source/Dockerfile
8283
context: .
8384
platforms: linux/amd64,linux/arm64
8485
push: true

‎.github/workflows/release-oura.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Create release files
2+
name: Release
3+
4+
on:
5+
release:
6+
types: [ published ]
7+
8+
env:
9+
DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
10+
11+
jobs:
12+
uploadBackend:
13+
# The type of runner that the job will run on
14+
runs-on: ubuntu-latest
15+
16+
# Steps represent a sequence of tasks that will be executed as part of the job
17+
steps:
18+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
19+
- uses: actions/checkout@v3
20+
- uses: actions/setup-java@v3
21+
with:
22+
distribution: temurin
23+
java-version: 11
24+
25+
- name: Gradle cache
26+
uses: actions/cache@v3
27+
with:
28+
# Cache gradle directories
29+
path: |
30+
~/.gradle/caches
31+
~/.gradle/wrapper
32+
# An explicit key for restoring and saving the cache
33+
key: ${{ runner.os }}-gradle-${{ hashFiles('gradlew', '**/*.gradle', 'gradle.properties', 'gradle/**') }}
34+
restore-keys: |
35+
${{ runner.os }}-gradle-
36+
37+
# Compile code
38+
- name: Compile code
39+
run: ./gradlew jar
40+
41+
# Upload it to GitHub
42+
- name: Upload to GitHub
43+
uses: AButler/[email protected]
44+
with:
45+
files: "*/build/libs/*"
46+
repo-token: ${{ secrets.GITHUB_TOKEN }}
47+
48+
# Build and push tagged release docker image
49+
docker:
50+
# The type of runner that the job will run on
51+
runs-on: ubuntu-latest
52+
53+
# Steps represent a sequence of tasks that will be executed as part of the job
54+
steps:
55+
- uses: actions/checkout@v3
56+
57+
# Setup docker build environment
58+
- name: Set up QEMU
59+
uses: docker/setup-qemu-action@v2
60+
- name: Set up Docker Buildx
61+
uses: docker/setup-buildx-action@v2
62+
63+
- name: Login to DockerHub
64+
uses: docker/login-action@v2
65+
with:
66+
username: ${{ secrets.DOCKERHUB_USERNAME }}
67+
password: ${{ secrets.DOCKERHUB_TOKEN }}
68+
69+
# Add Docker labels and tags
70+
- name: Docker meta
71+
id: docker_meta
72+
uses: docker/metadata-action@v4
73+
with:
74+
images: ${{ env.DOCKER_IMAGE }}
75+
tags: |
76+
type=semver,pattern={{version}}
77+
type=semver,pattern={{major}}.{{minor}}
78+
79+
- name: Build docker
80+
uses: docker/build-push-action@v3
81+
with:
82+
file: ./kafka-connect-oura-source/Dockerfile
83+
context: .
84+
platforms: linux/amd64,linux/arm64
85+
push: true
86+
tags: ${{ steps.docker_meta.outputs.tags }}
87+
# Use runtime labels from docker_meta_backend as well as fixed labels
88+
labels: |
89+
${{ steps.docker_meta.outputs.labels }}
90+
maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
91+
org.opencontainers.image.description=RADAR-base upload connector backend application
92+
org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
93+
org.opencontainers.image.vendor=RADAR-base
94+
org.opencontainers.image.licenses=Apache-2.0
95+
96+
- name: Inspect image
97+
run: |
98+
docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
99+
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ out/
66
.gradle/
77
docker/users
88
docker/source-fitbit.properties
9+
docker/source-oura.properties
10+
bin/
11+
.DS_Store

‎buildSrc/src/main/kotlin/Versions.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ object Versions {
1111
const val kafka = "$confluent-ce"
1212
const val avro = "1.11.0"
1313

14+
const val managementPortal = "2.0.0"
15+
1416
// From image
1517
const val jackson = "2.14.2"
1618

‎docker-compose.yml‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
version: '2.4'
2+
version: "2.4"
33

44
volumes:
55
fitbit-logs: {}
@@ -143,7 +143,9 @@ services:
143143
# RADAR Fitbit connector #
144144
#---------------------------------------------------------------------------#
145145
radar-fitbit-connector:
146-
build: .
146+
build:
147+
context: .
148+
dockerfile: ./kafka-connect-fitbit-source/Dockerfile
147149
image: radarbase/radar-connect-fitbit-source
148150
restart: on-failure
149151
volumes:
@@ -178,3 +180,44 @@ services:
178180
KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
179181
KAFKA_BROKERS: 3
180182
CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"
183+
184+
#---------------------------------------------------------------------------#
185+
# RADAR Oura connector #
186+
#---------------------------------------------------------------------------#
187+
radar-oura-connector:
188+
build:
189+
context: .
190+
dockerfile: ./kafka-connect-oura-source/Dockerfile
191+
image: radarbase/radar-connect-oura-source
192+
restart: on-failure
193+
volumes:
194+
- ./docker/source-oura.properties:/etc/kafka-connect/source-oura.properties
195+
- ./docker/users:/var/lib/kafka-connect-oura-source/users
196+
depends_on:
197+
- zookeeper-1
198+
- zookeeper-2
199+
- zookeeper-3
200+
- kafka-1
201+
- kafka-2
202+
- kafka-3
203+
- schema-registry-1
204+
environment:
205+
CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092
206+
CONNECT_REST_PORT: 8083
207+
CONNECT_GROUP_ID: "default"
208+
CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
209+
CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
210+
CONNECT_STATUS_STORAGE_TOPIC: "default.status"
211+
CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
212+
CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
213+
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
214+
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
215+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
216+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
217+
CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-oura-source/logs/connect.offsets"
218+
CONNECT_REST_ADVERTISED_HOST_NAME: "radar-oura-connector"
219+
CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
220+
CONNECTOR_PROPERTY_FILE_PREFIX: "source-oura"
221+
KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
222+
KAFKA_BROKERS: 3
223+
CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"

‎docker/source-oura.properties.template‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name=radar-oura-source
2+
connector.class=org.radarbase.connect.rest.oura.OuraSourceConnector
3+
tasks.max=4
4+
rest.source.base.url=https://api.ouraring.com
5+
rest.source.poll.interval.ms=5000
6+
oura.api.client=?
7+
oura.api.secret=?
8+
oura.user.repository.class=org.radarbase.connect.rest.oura.user.OuraServiceUserRepository
9+
oura.user.repository.url=
10+
oura.user.repository.client.id=radar_oura_connector
11+
oura.user.repository.client.secret=
12+
oura.user.repository.oauth2.token.url=

Dockerfile renamed to kafka-connect-fitbit-source/Dockerfile

File renamed without changes.

0 commit comments

Comments
 (0)