Skip to content

Commit 08166c1

Browse files
mneethirajmneethiraj
committed
ATLAS-5059: add support for rdbms backend and audit repository
1 parent 2d4b81b commit 08166c1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+2803
-90
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ jobs:
6969
cd dev-support/atlas-docker
7070
export DOCKER_BUILDKIT=1
7171
export COMPOSE_DOCKER_CLI_BUILD=1
72-
SKIPTESTS=false docker compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
72+
docker compose -f docker-compose.atlas-base.yml build
73+
SKIPTESTS=false docker compose -f docker-compose.atlas-build.yml up
7374
ATLAS_BUILD_CONTAINER=$(docker ps -a -q --filter "name=atlas-build")
7475
EXIT_CODE=$(docker inspect --format '{{.State.ExitCode}}' "$ATLAS_BUILD_CONTAINER")
7576
@@ -105,16 +106,11 @@ jobs:
105106
export DOCKER_BUILDKIT=1
106107
export COMPOSE_DOCKER_CLI_BUILD=1
107108
docker compose -f docker-compose.atlas-base.yml build
108-
docker compose \
109-
-f docker-compose.atlas.yml \
110-
-f docker-compose.atlas-hadoop.yml \
111-
-f docker-compose.atlas-hbase.yml \
112-
-f docker-compose.atlas-kafka.yml \
113-
-f docker-compose.atlas-hive.yml up -d --wait
109+
docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hive.yml up -d --wait
114110
115111
- name: Check status of containers and remove them
116112
run: |
117-
containers=(atlas atlas-hadoop atlas-hbase atlas-kafka atlas-hive);
113+
containers=(atlas-zk atlas-solr atlas-kafka atlas-db atlas-hadoop atlas-hbase atlas-hive atlas);
118114
flag=true;
119115
for container in "${containers[@]}"; do
120116
if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then

‎addons/kafka-bridge/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -337,12 +337,7 @@
337337
</resources>
338338
</configuration>
339339
</execution>
340-
</executions>
341-
</plugin>
342340

343-
<plugin>
344-
<artifactId>maven-resources-plugin</artifactId>
345-
<executions>
346341
<execution>
347342
<id>copy-solr-resources</id>
348343
<goals>

‎dev-support/atlas-docker/.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,5 @@ HBASE_VERSION=2.5.0
2424
KAFKA_VERSION=2.8.2
2525
HIVE_VERSION=3.1.3
2626
HIVE_HADOOP_VERSION=3.1.1
27+
28+
ATLAS_BACKEND=hbase

‎dev-support/atlas-docker/Dockerfile.atlas‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
FROM atlas-base:latest
1818

19+
ARG ATLAS_BACKEND
1920
ARG ATLAS_SERVER_JAVA_VERSION
2021
ARG ATLAS_VERSION
2122
ARG TARGETARCH

‎dev-support/atlas-docker/Dockerfile.atlas-db‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
FROM postgres:12
17+
FROM postgres:13.21
1818

1919
# Copy DB init script
2020
USER 0

‎dev-support/atlas-docker/README.md‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,22 @@ Docker files in this folder create docker images and run them to build Apache At
4444

4545
6. Build and deploy Apache Atlas in containers using docker-compose
4646

47-
6.1. Ensure that the `${HOME}/.m2` directory exists and Execute following command to build Apache Atlas:
47+
6.1. Build atlas-base image with the following commenr
48+
docker-compose -f docker-compose.atlas-base.yml build
49+
50+
6.2. Ensure that the `${HOME}/.m2` directory exists and Execute following command to build Apache Atlas:
4851
mkdir -p ${HOME}/.m2
49-
docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
52+
docker-compose docker-compose.atlas-build.yml up
5053

5154
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
5255

53-
6.2. Execute following command to install and start Atlas and dependent services (Solr, HBase, Kafka) in containers:
56+
6.3. To install and start Atlas using Postgres as backend store, execute following commands:
57+
export ATLAS_BACKEND=postgres
58+
docker-compose -f docker-compose.atlas.yml up -d --wait
5459

55-
docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml -f docker-compose.atlas-hbase.yml -f docker-compose.atlas-kafka.yml -f docker-compose.atlas-hive.yml up -d
60+
6.4. To install and start Atlas using HBase as backend store, execute following commands:
61+
export ATLAS_BACKEND=hbase
62+
docker-compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait
5663

5764
Apache Atlas will be installed at /opt/atlas/, and logs are at /var/logs/atlas directory.
5865

‎dev-support/atlas-docker/config/init_postgres.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,14 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
2323
CREATE USER hive WITH PASSWORD 'atlasR0cks!';
2424
CREATE DATABASE hive;
2525
GRANT ALL PRIVILEGES ON DATABASE hive TO hive;
26+
27+
CREATE USER atlas WITH PASSWORD 'atlasR0cks!';
28+
CREATE DATABASE atlas;
29+
GRANT ALL PRIVILEGES ON DATABASE atlas TO atlas;
30+
31+
\c hive
32+
GRANT ALL ON SCHEMA public TO public;
33+
34+
\c atlas
35+
GRANT ALL ON SCHEMA public TO public;
2636
EOSQL

‎dev-support/atlas-docker/docker-compose.atlas-backend.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
services:
2+
hbase:
3+
extends:
4+
file: docker-compose.atlas-hbase.yml
5+
service: atlas-hbase
6+
7+
postgres:
8+
extends:
9+
file: docker-compose.atlas-common.yml
10+
service: atlas-db

‎dev-support/atlas-docker/docker-compose.atlas-build.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ services:
1818
- ./patches:/home/atlas/patches
1919
- ./dist:/home/atlas/dist
2020
- ./../..:/home/atlas/src:delegated
21-
depends_on:
22-
- atlas-base
2321
environment:
2422
- ATLAS_VERSION
2523
- BRANCH

‎dev-support/atlas-docker/docker-compose.atlas-common.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
services:
2+
atlas-hadoop:
3+
build:
4+
context: .
5+
dockerfile: Dockerfile.atlas-hadoop
6+
args:
7+
- HADOOP_VERSION=${HADOOP_VERSION}
8+
image: atlas-hadoop
9+
container_name: atlas-hadoop
10+
hostname: atlas-hadoop.example.com
11+
stdin_open: true
12+
tty: true
13+
networks:
14+
- atlas
15+
ports:
16+
- "9000:9000"
17+
- "8088:8088"
18+
healthcheck:
19+
test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ]
20+
interval: 30s
21+
timeout: 10s
22+
retries: 30
23+
start_period: 40s
24+
environment:
25+
- HADOOP_VERSION
26+
27+
atlas-solr:
28+
build:
29+
context: .
30+
dockerfile: Dockerfile.atlas-solr
31+
image: atlas-solr
32+
container_name: atlas-solr
33+
hostname: atlas-solr.example.com
34+
networks:
35+
- atlas
36+
ports:
37+
- "8983:8983"
38+
39+
atlas-kafka:
40+
build:
41+
context: .
42+
dockerfile: Dockerfile.atlas-kafka
43+
args:
44+
- KAFKA_VERSION=${KAFKA_VERSION}
45+
- ATLAS_VERSION=${ATLAS_VERSION}
46+
image: atlas-kafka
47+
container_name: atlas-kafka
48+
hostname: atlas-kafka.example.com
49+
stdin_open: true
50+
tty: true
51+
networks:
52+
- atlas
53+
ports:
54+
- "9092:9092"
55+
depends_on:
56+
atlas-zk:
57+
condition: service_started
58+
environment:
59+
- KAFKA_VERSION
60+
- ATLAS_VERSION
61+
62+
atlas-zk:
63+
build:
64+
context: .
65+
dockerfile: Dockerfile.atlas-zk
66+
image: atlas-zk
67+
container_name: atlas-zk
68+
hostname: atlas-zk.example.com
69+
networks:
70+
- atlas
71+
ports:
72+
- "2181:2181"
73+
74+
atlas-db:
75+
build:
76+
context: .
77+
dockerfile: Dockerfile.atlas-db
78+
image: atlas-db
79+
container_name: atlas-db
80+
hostname: atlas-db.example.com
81+
networks:
82+
- atlas
83+
healthcheck:
84+
test: 'su -c "pg_isready -q" postgres'
85+
interval: 30s
86+
timeout: 10s
87+
retries: 30
88+
start_period: 40s
89+

0 commit comments

Comments
 (0)