Skip to content

Commit f82a147

Browse files
author
Jay Moran
committed
rpi production release
2 parents dbfeb47 + 5df840f commit f82a147

File tree

10 files changed

+420
-18
lines changed

10 files changed

+420
-18
lines changed

‎.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/sh
22
# Docker production build environment
3-
TAG=0.19a0
3+
TAG=rpi-0.18

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,4 @@ share/python-wheels/
5858
.installed.cfg
5959
*.egg
6060
MANIFEST
61+
.vscode

‎docker-compose-raspberrypi-production.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Build production images
2+
version: '3'
3+
4+
services:
5+
lab:
6+
build:
7+
context: .
8+
dockerfile: docker/lab/rpi_production.Dockerfile
9+
image: "aliro_lab:${TAG}"
10+
tty: true
11+
stdin_open: true
12+
volumes:
13+
- "/appsrc"
14+
- "./data/datasets/user:/appsrc/data/datasets/user"
15+
- "./data/recommenders/pennaiweb:/appsrc/data/recommenders/pennaiweb"
16+
- "./target:/appsrc/target"
17+
ports:
18+
- "5080:5080"
19+
env_file:
20+
- ./config/common.env
21+
- ./config/ai.env
22+
depends_on:
23+
- dbmongo
24+
25+
machine:
26+
build:
27+
context: .
28+
dockerfile: docker/machine/rpi_production.Dockerfile
29+
image: "aliro_machine:${TAG}"
30+
tty: true
31+
stdin_open: true
32+
volumes:
33+
- "/appsrc"
34+
- "./target:/appsrc/target"
35+
ports:
36+
- "5081:5081"
37+
env_file: ./config/common.env
38+
environment:
39+
- MACHINE_PORT=5081
40+
depends_on:
41+
- lab
42+
- dbmongo
43+
44+
dbmongo:
45+
build:
46+
context: .
47+
dockerfile: docker/dbmongo/rpi.Dockerfile
48+
image: "aliro_dbmongo:${TAG}"
49+
tty: true
50+
stdin_open: true
51+
ports:
52+
- "27017:27017"
53+
env_file: ./config/common.env

‎docker/lab/rpi.Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ FROM ubuntu:bionic
1111
RUN groupadd --gid 1000 node \
1212
&& useradd --uid 1000 --gid node --shell /bin/bash --create-home node
1313

14-
ENV NODE_VERSION 12.20.0
14+
ENV NODE_VERSION 12.22.9
1515

1616
RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
1717
&& case "${dpkgArch##*-}" in \

‎docker/lab/rpi_production.Dockerfile‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
FROM ubuntu:bionic
2+
3+
# copy src for lab
4+
COPY lab /appsrc/lab
5+
COPY ai /appsrc/ai
6+
7+
#RUN apt-get update --fix-missing && apt-get install -y wget
8+
9+
#nodejs
10+
#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \
11+
# tar -xvf ~/node.tar.xz -C /opt/ && \
12+
# rm ~/node.tar.xz
13+
#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH
14+
15+
RUN groupadd --gid 1000 node \
16+
&& useradd --uid 1000 --gid node --shell /bin/bash --create-home node
17+
18+
ENV NODE_VERSION 12.22.9
19+
20+
RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
21+
&& case "${dpkgArch##*-}" in \
22+
amd64) ARCH='x64';; \
23+
ppc64el) ARCH='ppc64le';; \
24+
s390x) ARCH='s390x';; \
25+
arm64) ARCH='arm64';; \
26+
armhf) ARCH='armv7l';; \
27+
i386) ARCH='x86';; \
28+
*) echo "unsupported architecture"; exit 1 ;; \
29+
esac \
30+
&& set -ex \
31+
# libatomic1 for arm
32+
&& apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \
33+
&& rm -rf /var/lib/apt/lists/* \
34+
&& for key in \
35+
4ED778F539E3634C779C87C6D7062848A1AB005C \
36+
94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
37+
1C050899334244A8AF75E53792EF661D867B9DFA \
38+
71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
39+
8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
40+
C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
41+
C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \
42+
DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
43+
A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
44+
108F52B48DB57BB0CC439B2997B01419BD92F80A \
45+
B9E2F5981AA6E0CD28160D9FF13993A75599653C \
46+
; do \
47+
gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \
48+
gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \
49+
gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
50+
gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
51+
gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \
52+
done \
53+
&& curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
54+
&& curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
55+
&& gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
56+
&& grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
57+
&& tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
58+
&& rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
59+
&& apt-mark auto '.*' > /dev/null \
60+
&& find /usr/local -type f -executable -exec ldd '{}' ';' \
61+
| awk '/=>/ { print $(NF-1) }' \
62+
| sort -u \
63+
| xargs -r dpkg-query --search \
64+
| cut -d: -f1 \
65+
| sort -u \
66+
| xargs -r apt-mark manual \
67+
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
68+
&& ln -s /usr/local/bin/node /usr/local/bin/nodejs \
69+
# smoke tests
70+
&& node --version \
71+
&& npm --version
72+
73+
ENV YARN_VERSION 1.22.5
74+
75+
RUN set -ex \
76+
&& savedAptMark="$(apt-mark showmanual)" \
77+
&& apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \
78+
&& rm -rf /var/lib/apt/lists/* \
79+
&& for key in \
80+
6A010C5166006599AA17F08146C2130DFD2497F5 \
81+
; do \
82+
gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \
83+
gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \
84+
gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
85+
gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
86+
gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \
87+
done \
88+
&& curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \
89+
&& curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \
90+
&& gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
91+
&& mkdir -p /opt \
92+
&& tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \
93+
&& ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \
94+
&& ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \
95+
&& rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
96+
&& apt-mark auto '.*' > /dev/null \
97+
&& { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \
98+
&& find /usr/local -type f -executable -exec ldd '{}' ';' \
99+
| awk '/=>/ { print $(NF-1) }' \
100+
| sort -u \
101+
| xargs -r dpkg-query --search \
102+
| cut -d: -f1 \
103+
| sort -u \
104+
| xargs -r apt-mark manual \
105+
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
106+
# smoke test
107+
&& yarn --version
108+
109+
ARG docker_filepath=docker/lab/files
110+
ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel
111+
112+
RUN apt-get update --fix-missing && apt-get install -y \
113+
openssh-client openssh-server telnet apache2 \
114+
net-tools iputils-ping xz-utils \
115+
ngrep ca-cacert \
116+
build-essential cmake lsb-core cpio mesa-common-dev \
117+
dos2unix curl git libatlas-base-dev gfortran \
118+
--no-install-recommends && \
119+
apt-get clean && \
120+
rm -rf /var/lib/apt/lists/*
121+
122+
# Ensure specific python version
123+
# See: https://stackoverflow.com/a/58562728/1730417
124+
# and: https://askubuntu.com/a/1176271/260220
125+
RUN apt-get update && apt-get install -y software-properties-common
126+
RUN add-apt-repository ppa:deadsnakes/ppa
127+
RUN apt-get update
128+
RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils
129+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1
130+
RUN update-alternatives --set python /usr/bin/python3.7
131+
RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
132+
python get-pip.py --force-reinstall && \
133+
rm get-pip.py
134+
135+
# ensure local python is preferred over distribution python
136+
ENV PATH /usr/local/bin:$PATH
137+
138+
# http://bugs.python.org/issue19846
139+
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
140+
ENV LANG C.UTF-8
141+
142+
ENV TZ=America/New_York
143+
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
144+
ARG DEBIAN_FRONTEND=noninteractive
145+
RUN apt-get update && apt-get install -y --no-install-recommends \
146+
libbluetooth-dev \
147+
tk-dev \
148+
uuid-dev \
149+
&& rm -rf /var/lib/apt/lists/*
150+
RUN apt-get update && apt-get install -y --no-install-recommends \
151+
wget python3.7-venv python3-numpy \
152+
&& apt-get clean \
153+
&& rm -rf /var/lib/apt/list/*
154+
155+
RUN python3 --version
156+
RUN python --version
157+
RUN python -m venv /venv
158+
ENV PATH=/venv/bin:$PATH
159+
RUN python --version
160+
RUN pip --version
161+
162+
# setup python environment
163+
COPY ${docker_filepath}/requirements-raspberrypi.txt /root/
164+
RUN mkdir /root/wheel
165+
COPY ${python_wheel_directory}/*.whl /root/wheel/
166+
RUN pip3 install --no-cache-dir cython
167+
RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt
168+
### bill's surprise fork
169+
# NOTE: For some reason, this install doesn't seem to recognize the installed numpy
170+
# when run on Docker. As a workaround, the `wheel` directory includes a wheel
171+
# for this install.
172+
#RUN pip3 install --no-cache-dir git+https://github.com/lacava/[email protected]
173+
RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise
174+
175+
## Webserver
176+
177+
RUN rm /etc/apache2/sites-enabled/*
178+
COPY ${docker_filepath}/ports.conf /etc/apache2/
179+
RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \
180+
&& cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \
181+
&& cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \
182+
&& cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \
183+
&& cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \
184+
&& cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \
185+
&& cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \
186+
&& cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled
187+
188+
# setup node environment
189+
RUN npm install -g pm2 webpack --silent --progress=false
190+
191+
# install lab/webapp/node_modules to an anon volume
192+
WORKDIR /appsrc/lab/webapp
193+
COPY lab/webapp/package.json /appsrc/lab/webapp/
194+
COPY lab/webapp/package-lock.json /appsrc/lab/webapp/
195+
RUN dos2unix /appsrc/lab/webapp/*.json
196+
RUN npm install --silent --progress=false
197+
198+
# install lab/node_modules to an anon volume
199+
WORKDIR /appsrc/lab
200+
COPY lab/package.json /appsrc/lab/
201+
COPY lab/package-lock.json /appsrc/lab/
202+
RUN dos2unix /appsrc/lab/*.json
203+
RUN npm install --silent --progress=false
204+
205+
RUN apt-get install -y libgfortran5 liblapack-dev
206+
207+
COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/
208+
COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd
209+
COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/
210+
211+
WORKDIR /root/
212+
213+
# Webserver - paiwww
214+
COPY ${docker_filepath}/start.sh /root/
215+
216+
## Utility script, used when starting ai
217+
COPY ${docker_filepath}/wait-for-it.sh /root/
218+
RUN ["chmod", "+x", "/root/wait-for-it.sh"]
219+
220+
221+
## PennAI Lab server
222+
COPY ${docker_filepath}/entrypoint.sh /root/
223+
RUN ["chmod", "+x", "/root/entrypoint.sh"]
224+
225+
RUN dos2unix /root/start.sh \
226+
&& dos2unix /root/wait-for-it.sh \
227+
&& dos2unix /root/entrypoint.sh
228+
229+
# copy knowledgebase
230+
COPY 'data/knowledgebases/sklearn-benchmark-data-knowledgebase-r6.tsv.gz' /appsrc/data/knowledgebases/
231+
COPY 'data/knowledgebases/pmlb_classification_metafeatures.csv.gz' /appsrc/data/knowledgebases/
232+
COPY 'data/knowledgebases/pmlb_regression_results.pkl.gz' /appsrc/data/knowledgebases/
233+
COPY 'data/knowledgebases/pmlb_regression_metafeatures.csv.gz' /appsrc/data/knowledgebases/
234+
235+
# set version and build environment; tag.sh is sourced in entrypoint.sh
236+
ENV BUILD_ENV='prod'
237+
COPY .env /etc/profile.d/
238+
RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh'
239+
RUN dos2unix /etc/profile.d/tag.sh
240+
RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh
241+
242+
243+
# Start the webserver
244+
CMD ["/bin/bash", "/root/start.sh"]
245+
246+
# EXPOSE 443
247+
EXPOSE 5080
248+
WORKDIR /appsrc/lab/
249+
ENTRYPOINT ["/root/entrypoint.sh"]

‎docker/machine/rpi.Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM python:3.7-buster
22

3-
ENV NODE_VERSION 12.20.0
3+
ENV NODE_VERSION 12.22.9
44

55
RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
66
&& case "${dpkgArch##*-}" in \

0 commit comments

Comments
 (0)