Skip to content

Commit 41e6869

Browse files
mmolsmmols
authored
Add docker compose setup for development / testing (#284)
* add sample Dockerfile as basis for compose setup supporting sshd * add compose file with build container updates * add separate dockerfiles for ubuntu/rocky * proxy files from all repos * remove extra logging * simplify setup and add README * add separate BUILD_OS_ARCH and BUILD_OS_FLAVOR * add backups volume mount * add make node-ips target
1 parent c483af8 commit 41e6869

File tree

12 files changed

+452
-25
lines changed

12 files changed

+452
-25
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ devel/pg/postgresml/
77

88
devel/repo/cleanREMOTE
99
devel/repo/local
10-
10+
devel/setup/compose/repo
1111
devel/mqttclient/
1212

1313
src/packages/*.rpm

‎devel/setup/10-toolset.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,6 @@ if [ $rc == "0" ]; then
9696

9797
install-apt-toolset
9898
fi
99+
100+
exit 0
99101

‎devel/setup/Dockerfile‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎devel/setup/Dockerfile.rocky810‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
FROM rockylinux/rockylinux:8.10
2+
RUN touch /etc/hostname
3+
4+
SHELL ["/bin/bash", "-c"]
5+
6+
USER root
7+
8+
ENV install="dnf install -y --allowerasing"
9+
RUN $install dnf-plugins-core
10+
RUN $install python3 python3-pip git wget curl pigz which zip sqlite
11+
RUN $install openssh-server systemd sudo
12+
13+
RUN useradd build -U -m -d /home/build \
14+
&& echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
15+
16+
USER build
17+
WORKDIR /home/build/dev
18+
19+
RUN git clone https://github.com/pgedge/cli
20+
# Temporary workaround for script behavior
21+
RUN cd cli/devel/setup && bash -x ./10-toolset.sh || true
22+
RUN cd cli/devel/setup && ./20-venv.sh
23+
RUN cd cli/devel/setup && ./30-bashrc.sh
24+
RUN source ~/.bashrc
25+
RUN cd cli/devel/setup && ./40-awscli.sh
26+
27+
CMD /bin/bash

‎devel/setup/Dockerfile.rocky95‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
FROM rockylinux/rockylinux:9.5
2+
RUN touch /etc/hostname
3+
4+
SHELL ["/bin/bash", "-c"]
5+
6+
USER root
7+
8+
ENV install="dnf install -y --allowerasing"
9+
RUN $install dnf-plugins-core
10+
RUN $install python3 python3-pip git wget curl pigz which zip sqlite
11+
RUN $install openssh-server systemd sudo
12+
13+
RUN useradd build -U -m -d /home/build \
14+
&& echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
15+
16+
USER build
17+
WORKDIR /home/build/dev
18+
19+
RUN git clone https://github.com/pgedge/cli
20+
# Temporary workaround for script behavior
21+
RUN cd cli/devel/setup && bash -x ./10-toolset.sh || true
22+
RUN cd cli/devel/setup && ./20-venv.sh
23+
RUN cd cli/devel/setup && ./30-bashrc.sh
24+
RUN source ~/.bashrc
25+
RUN cd cli/devel/setup && ./40-awscli.sh
26+
27+
CMD /bin/bash

‎devel/setup/compose/Dockerfile.node.rocky95‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
FROM rockylinux/rockylinux:9.5
2+
3+
RUN touch /etc/hostname
4+
RUN dnf -y update && dnf remove -y subscription-manager dnf-plugin-subscription-manager
5+
6+
ENV install="dnf install -y --allowerasing"
7+
RUN $install python3 python3-pip git wget curl pigz which zip sqlite postgresql
8+
RUN $install openssh-server systemd sudo
9+
10+
11+
RUN adduser --home-dir /home/pgedge --shell /bin/bash pgedge
12+
RUN echo '%pgedge ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
13+
RUN rm -f /var/run/nologin
14+
RUN sed -i 's/^#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
15+
RUN sed -i 's/^PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
16+
RUN ssh-keygen -A
17+
RUN /usr/sbin/sshd
18+
19+
RUN mkdir -p /backups
20+
RUN chown pgedge:pgedge /backups
21+
22+
USER pgedge
23+
WORKDIR /home/pgedge
24+
RUN mkdir -p .ssh/
25+
RUN ssh-keygen -t rsa -b 4096 -N "" -f /home/pgedge/.ssh/id_rsa
26+
27+
RUN cat /home/pgedge/.ssh/id_rsa.pub >> /home/pgedge/.ssh/authorized_keys
28+
RUN chmod 600 /home/pgedge/.ssh/authorized_keys
29+
RUN chmod 700 /home/pgedge/.ssh/
30+
31+
RUN sudo chown pgedge:pgedge /home/pgedge/.ssh/id_rsa
32+
RUN sudo chmod 600 /home/pgedge/.ssh/id_rsa
33+
34+
RUN sed -i '1iexport PGBACKREST_REPO1_CIPHER_PASS=Really_s3cure_password' ~/.bashrc
35+
36+
USER root
37+
EXPOSE 22
38+
39+
CMD ["/usr/sbin/sshd", "-D"]
40+

‎devel/setup/compose/Dockerfile.node.ubuntu2204‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
FROM ubuntu:22.04
2+
3+
ENV TZ=America/New_York
4+
ENV DEBIAN_FRONTEND=noninteractive
5+
RUN apt-get update && apt-get remove -y --purge snapd && apt-get -y upgrade
6+
7+
ENV install="apt-get install -y --no-install-recommends"
8+
RUN $install python3 python3-pip git wget curl pigz zip sqlite3 postgresql
9+
RUN $install openssh-server sudo systemd
10+
11+
RUN adduser --home /home/pgedge --shell /bin/bash pgedge
12+
RUN echo '%pgedge ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
13+
RUN rm -f /var/run/nologin
14+
RUN sed -i 's/^#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
15+
RUN sed -i 's/^#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
16+
RUN ssh-keygen -A
17+
RUN mkdir -p /run/sshd && /usr/sbin/sshd
18+
19+
RUN mkdir -p /backups
20+
RUN chown pgedge:pgedge /backups
21+
22+
USER pgedge
23+
WORKDIR /home/pgedge
24+
RUN mkdir -p .ssh/
25+
RUN ssh-keygen -t rsa -b 4096 -N "" -f /home/pgedge/.ssh/id_rsa
26+
27+
RUN cat /home/pgedge/.ssh/id_rsa.pub >> /home/pgedge/.ssh/authorized_keys
28+
RUN chmod 600 /home/pgedge/.ssh/authorized_keys
29+
RUN chmod 700 /home/pgedge/.ssh/
30+
31+
RUN sudo chown pgedge:pgedge /home/pgedge/.ssh/id_rsa
32+
RUN sudo chmod 600 /home/pgedge/.ssh/id_rsa
33+
34+
RUN echo 'export PGBACKREST_REPO1_CIPHER_PASS=Really_s3cure_password' >> ~/.bashrc
35+
36+
USER root
37+
EXPOSE 22
38+
39+
CMD ["/usr/sbin/sshd", "-D"]

‎devel/setup/compose/Dockerfile.repo‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM rockylinux/rockylinux:9-ubi
2+
RUN touch /etc/hostname
3+
4+
# Set the working directory
5+
WORKDIR /usr/src/
6+
7+
# Ensure /usr/src/repo is readable by the HTTP server
8+
RUN mkdir -p /usr/src/repo
9+
RUN chmod 755 /usr/src/repo
10+
11+
ADD proxy_server.py /usr/src/proxy_server.py
12+
13+
# Install required packages
14+
RUN dnf update -y && dnf install -y \
15+
python3-pip \
16+
&& dnf clean all
17+
18+
# Install the HTTP server package
19+
RUN pip3 install httpserver
20+
21+
# Expose the port the server will run on
22+
EXPOSE 8000
23+
24+
# Command to run the HTTP server
25+
CMD ["python3", "-m", "proxy_server", "--port", "8000", "--directory", "/usr/src/repo"]

‎devel/setup/compose/Makefile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
2+
.DEFAULT_GOAL := run
3+
4+
NUM_NODES ?= 2
5+
OS_ARCH ?= linux/$(shell uname -m)
6+
# Supports rocky95, ubuntu2204
7+
OS_FLAVOR ?= rocky95
8+
REPO ?= http://repo:8000/download
9+
10+
BUILD_OS_ARCH ?= linux/$(shell uname -m)
11+
# Supports rocky95, rocky810
12+
BUILD_OS_FLAVOR ?= rocky95
13+
14+
COMPOSE_ARGS := NUM_NODES=$(NUM_NODES) OS_ARCH=$(OS_ARCH) OS_FLAVOR=$(OS_FLAVOR) BUILD_OS_ARCH=$(BUILD_OS_ARCH) BUILD_OS_FLAVOR=$(BUILD_OS_FLAVOR) REPO=$(REPO)
15+
16+
.PHONY: compose-up
17+
compose-up:
18+
$(COMPOSE_ARGS) docker compose build && $(COMPOSE_ARGS) docker compose up -d
19+
20+
.PHONY: compose-down
21+
compose-down:
22+
docker compose down
23+
24+
.PHONY: exec-build
25+
exec-build:
26+
@docker exec -it pgedge-build-1 bash
27+
28+
define EXEC_NODE
29+
.PHONY: exec-n$(1)
30+
exec-n$(1):
31+
@docker exec -u pgedge -it pgedge-node-$(1) bash
32+
endef
33+
34+
.PHONY: node-ips
35+
node-ips:
36+
@docker compose ps -q node | xargs docker inspect -f '{{.Name}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'
37+
38+
$(foreach n,1 2 3 4 5 6 7 8 9 10,$(eval $(call EXEC_NODE,$n)))

‎devel/setup/compose/README.md‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# Compose Setup for pgEdge Development
2+
3+
This setup allows you to build and run a multi-node pgEdge setup using Docker Compose. Below are the instructions and available arguments to customize your setup.
4+
5+
## Prerequisites
6+
7+
- Docker
8+
- Docker Compose
9+
10+
## Files
11+
12+
- `docker-compose.yaml`: Defines the services and configurations for the Docker Compose setup.
13+
- `Makefile`: Contains the commands to manage the Docker Compose setup.
14+
- `proxy_server.py`: Contains the code which enables the repo proxy container
15+
- `Dockerfile.node.<os>`: Dockerfiles for each supported operating system
16+
- `Dockerfile.repo`: Dockerfile for the repo proxy server
17+
18+
## Usage
19+
20+
### Starting the Environment
21+
22+
To start the environment, run the following command:
23+
24+
```sh
25+
make compose-up
26+
```
27+
28+
This will build and start the services defined in the `docker-compose.yaml` file in Docker.
29+
30+
### Stopping the Environment
31+
32+
To stop the environment, run:
33+
34+
```sh
35+
make compose-down
36+
```
37+
38+
### Accessing the build container
39+
40+
The build container is where you make code changes and build packages.
41+
42+
To access the build container, run:
43+
44+
```sh
45+
make exec-build
46+
```
47+
48+
### Configuring the build container
49+
50+
The build container is setup the same as a standard development environment outside of Docker.
51+
52+
Required credentials (AWS, Git) are not configured on the build container.
53+
54+
You must configure these after starting the environment, and run `50-pull-IN.sh` if needed for your work.
55+
56+
### Understanding the repo container
57+
58+
The repo container serves as a local http server inside the network for fetching CLI packages.
59+
60+
It's setup as a proxy, where the default behavior is to serve a local package if it exists.
61+
62+
If a local package does not exist, it will fallback to a package available in the chosen REPO.
63+
64+
The chosen repo corresponds to the URL that you choose to configure:
65+
66+
- http://repo:8000/download corresponds with https://pgedge-download.s3.amazonaws.com/REPO
67+
- http://repo:8000/upstream corresponds with https://pgedge-upstream.s3.amazonaws.com/REPO
68+
- http://repo:8000/devel corresponds with https://pgedge-devel.s3.amazonaws.com/REPO
69+
70+
The `out` directory within the build container is mounted in the repo container to enable this setup.
71+
72+
You can monitor the behavior of the proxy via `docker logs pgedge-repo-1`
73+
74+
### Building the CLI packages
75+
76+
A common workflow in this setup is to leverage already built packages to increase development velocity.
77+
78+
If you are only working on the CLI code, you can typically just build that component:
79+
80+
```
81+
source env.sh
82+
./build.sh -X posix -c $bundle-cli -N $hubV
83+
```
84+
85+
From there, utilize the node containers to test out your changes. Other packages will be pulled from the configured REPO.
86+
87+
### Accessing node containers
88+
89+
node containers are used to test changes you have developed within the build container.
90+
91+
You can access individual node containers using the following commands:
92+
93+
```sh
94+
make exec-n1
95+
make exec-n2
96+
# ... up to make exec-n10
97+
```
98+
99+
Once connected, you can setup the pgEdge CLI using the repo container:
100+
101+
```
102+
wget $REPO/install.py
103+
python3 install.py
104+
cd pgedge
105+
```
106+
107+
## Networking
108+
109+
All containers in the compose setup are attached to the same docker bridge network. This makes it easier to connect across nodes using hostnames.
110+
111+
- `repo` and `build` can be accessed directly by these hostnames since there are no replicas
112+
- nodes can be accessed via hostnames prefixed with the compose name `pgedge-node-1`
113+
114+
## Available arguments
115+
116+
You can customize the setup using the following arguments:
117+
118+
- `NUM_NODES`: Number of node containers to deploy. Default is `2`.
119+
- `OS_ARCH`: The OS architecture for the node containers. Default is `linux/$(uname -m)`.
120+
- `OS_FLAVOR`: The OS flavor for the node containers. Supported values are `rocky95` and `ubuntu2204`. Default is `rocky95`.
121+
- `BUILD_OS_FLAVOR`: The OS architecture for the build container. Default is `linux/$(uname -m)`.
122+
- `BUILD_OS_ARCH`: The OS flavor for the build container. Supported values are `rocky95` and `rocky810`. Default is `rocky95`.
123+
- `REPO`: The repository URL for downloading dependencies. Default is `http://repo:8000/download`.
124+
125+
### Example
126+
127+
To start the environment with 3 nodes and using `ubuntu2204` as the OS flavo, with the REPO set to upstream, run:
128+
129+
```sh
130+
REPO=http://repo:8000/upstream NUM_NODES=3 OS_FLAVOR=ubuntu2204 make compose-up
131+
```
132+
133+
## Notes
134+
135+
- Passwordless SSH is setup between the nodes to make it easier to test the cluster module
136+
- The `PGBACKREST_REPO1_CIPHER_PASS` environment variable is set to `supersecret` by default. Change it as needed.

0 commit comments

Comments
 (0)