Edit Dockerfile for new tools

m09 · m09 · commit cdf6b63d481d · 2020-01-27T12:38:20.000+01:00
Signed-off-by: m09 &lt;142691+m09@users.noreply.github.com&gt;
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,4 @@
-repos
-notebooks
+env.sh
+.mypy_cache
+notebooks/output
+notebooks/repos
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,4 @@
-repos
+env.sh
 .mypy_cache
+notebooks/output
+notebooks/repos
diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,14 @@
-FROM ubuntu:18.04
+FROM python:3.7.6-slim-buster
 
-RUN mkdir -p /devfest/repos /devfest/conf /devfest/nltk_data /devfest/notebooks
+WORKDIR /workdir
 
-WORKDIR /devfest
+RUN mkdir -p repos conf scripts nltk_data notebooks
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-ENV BBLFSH_HOSTNAME amld_bblfshd
-ENV BBLFSH_PORT 9432
-ENV GITBASE_HOSTNAME amld_gitbase
-ENV GITBASE_PORT 3306
-ENV GITBASE_USERNAME root
-ENV GITBASE_PASSWORD ""
 ENV ARTM_SHARED_LIBRARY /usr/local/lib/libartm.so
-ENV NLTK_DATA /devfest/nltk_data
+ENV NLTK_DATA /workdir/nltk_data
+ENV TREE_SITTER_LANGUAGES_SO /workdir/tree-sitter-languages.so
 ENV LANG en_US.UTF-8
 ENV LC_ALL en_US.UTF-8
 
@@ -30,56 +25,31 @@ RUN apt-get update \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-COPY requirements-bigartm.txt conf
+COPY conf/requirements-bigartm.txt conf/
+COPY conf/requirements-tree-sitter.txt conf/
+COPY scripts/install-bigartm scripts/
+COPY scripts/*tree-sitter* scripts/
+COPY scripts/install-nltk-data scripts/
 
-RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
-  build-essential \
-  cmake \
-  git \
-  libboost-chrono-dev \
-  libboost-date-time-dev \
-  libboost-dev \
-  libboost-filesystem-dev \
-  libboost-iostreams-dev \
-  libboost-program-options-dev \
-  libboost-system-dev \
-  libboost-thread-dev \
-  libboost-timer-dev \
-  make \
-  python3-dev \
-  unzip \
-  && ln -s /usr/bin/python3 /usr/local/bin/python \
-  && curl https://bootstrap.pypa.io/get-pip.py | python \
-  && pip3 install --no-cache-dir -r conf/requirements-bigartm.txt \
-  && git clone --branch v0.10.0 https://github.com/bigartm/bigartm.git /opt/bigartm \
-  && mkdir /opt/bigartm/build \
-  && cd /opt/bigartm/build \
-  && cmake -DINSTALL_PYTHON_PACKAGE=ON -DPYTHON=python3 .. \
-  && make -j$(getconf _NPROCESSORS_ONLN) \
-  && make install \
-  && rm -rf /usr/share/doc /usr/share/man \
-  && apt-get autoremove --purge -y \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
-  && mkdir $NLTK_DATA/corpora \
-  && curl -sSL https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip -o $NLTK_DATA/corpora/stopwords.zip \
-  && unzip $NLTK_DATA/corpora/stopwords.zip -d $NLTK_DATA/corpora/ \
-  && rm $NLTK_DATA/corpora/stopwords.zip \
+RUN ./scripts/install-bigartm \
+  && ./scripts/install-tree-sitter \
+  && ./scripts/install-nltk-data \
   && pip3 install --no-cache-dir "jupyter == 1.0.0" \
   && pip3 install --no-cache-dir \
   "jupyter_contrib_nbextensions == 0.5.1" \
   "jupyter_nbextensions_configurator == 0.4.1" \
   && jupyter contrib nbextension install \
   && jupyter nbextensions_configurator enable
 
-COPY jupyter-notebook-config.json /root/.jupyter/nbconfig/notebook.json
-COPY jupyter-server-config.json /root/.jupyter/jupyter_notebook_config.json
+COPY conf/jupyter-notebook-config.json /root/.jupyter/nbconfig/notebook.json
+COPY conf/jupyter-server-config.json /root/.jupyter/jupyter_notebook_config.json
+
+COPY conf/requirements-setup.txt conf/
+COPY conf/requirements.txt conf/
 
-COPY requirements*.txt conf/
-RUN pip3 install --no-cache-dir -r conf/requirements.txt \
-  && pip3 install --no-cache-dir -r conf/requirements-tf.txt
+RUN pip3 install --no-cache-dir -r conf/requirements-setup.txt \
+  && pip3 install --no-cache-dir -r conf/requirements.txt
 
-WORKDIR /devfest/notebooks
+WORKDIR /workdir/notebooks
 
 ENTRYPOINT jupyter notebook --ip 0.0.0.0 --allow-root --no-browser
diff --git a/Makefile b/Makefile
@@ -1,52 +1,25 @@
 run: jupyter
+.PHONY: run
 
 build-and-run:
 	$(MAKE) jupyter-image
 	$(MAKE) jupyter
+.PHONY: build-and-run
 
 stop:
-	docker stop devfest_jupyter devfest_gitbase devfest_bblfshd \
-		> /dev/null 2>&1 || true
-
-bblfshd:
-	docker start devfest_bblfshd > /dev/null 2>&1 \
-		|| docker run \
-			--detach \
-			--rm \
-			--name devfest_bblfshd \
-			--privileged \
-			--publish 9432:9432 \
-			--memory 2G \
-			bblfsh/bblfshd:v2.15.0-drivers \
-			--log-level DEBUG
-
-gitbase: bblfshd
-	docker start devfest_gitbase > /dev/null 2>&1 \
-		|| docker run \
-			--detach \
-			--rm \
-			--name devfest_gitbase \
-			--publish 3306:3306 \
-			--link devfest_bblfshd:devfest_bblfshd \
-			--env BBLFSH_ENDPOINT=devfest_bblfshd:9432 \
-			--env MAX_MEMORY=1024 \
-			--volume $(PWD)/repos/git-data:/opt/repos \
-			srcd/gitbase:v0.24.0-rc2
+	docker stop mloncode/workshop > /dev/null 2>&1 || true
+.PHONY: stop
 
 jupyter-image:
-	docker build -t mloncode/devfest .
+	docker build -t mloncode/workshop .
+.PHONY: jupyter-image
 
-jupyter: gitbase bblfshd
-	docker start devfest_jupyter > /dev/null 2>&1 \
+jupyter:
+	docker start mloncode/workshop > /dev/null 2>&1 \
 		|| docker run \
 		    --rm \
-		    --name devfest_jupyter \
+		    --name mloncode/workshop \
 		    --publish 8888:8888 \
-		    --link devfest_bblfshd:devfest_bblfshd \
-		    --link devfest_gitbase:devfest_gitbase \
-		    --volume $(PWD)/notebooks:/devfest/notebooks \
-		    --volume $(PWD)/repos:/devfest/repos \
-		    mloncode/devfest
-
-
-.PHONY: run build-and-run stop bblfshd gitbase jupyter-image jupyter
+		    --volume $(PWD)/notebooks:/workdir/notebooks \
+		    mloncode/workshop
+.PHONY: jupyter
diff --git a/conf/jupyter-notebook-config.json b/conf/jupyter-notebook-config.json
diff --git a/conf/jupyter-server-config.json b/conf/jupyter-server-config.json
diff --git a/conf/requirements-bigartm.in b/conf/requirements-bigartm.in
diff --git a/conf/requirements-bigartm.txt b/conf/requirements-bigartm.txt
@@ -0,0 +1,18 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements-bigartm.in
+#
+numpy==1.17.2
+pandas==0.25.1
+protobuf==3.10.0
+python-dateutil==2.8.1    # via pandas
+pytz==2019.3              # via pandas
+scipy==1.3.1
+six==1.14.0               # via protobuf, python-dateutil
+tqdm==4.36.1
+wheel==0.33.6
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/conf/requirements-setup.txt b/conf/requirements-setup.txt
@@ -0,0 +1 @@
+Cython == 0.29.14
diff --git a/conf/requirements-tree-sitter.in b/conf/requirements-tree-sitter.in
@@ -0,0 +1 @@
+tree-sitter == 0.0.5
diff --git a/conf/requirements-tree-sitter.txt b/conf/requirements-tree-sitter.txt
@@ -0,0 +1,7 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile conf/requirements-tree-sitter.in
+#
+tree-sitter==0.0.5
diff --git a/conf/requirements.in b/conf/requirements.in
@@ -1,8 +1,8 @@
-PyMySQL == 0.9.3
+coloredlogs == 10.0
 requests == 2.22.0
-bblfsh == 4.1.0
-GitPython == 3.0.3
+pygit2 == 1.0.2
 PyStemmer == 1.3.0
 pyLDAvis == 2.1.2
 scikit-learn == 0.21.3
 seriate == 1.1.1
+tqdm == 4.36.1
diff --git a/conf/requirements.txt b/conf/requirements.txt
@@ -0,0 +1,49 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile conf/requirements.in
+#
+attrs==19.3.0             # via pytest
+certifi==2019.11.28       # via requests
+cffi==1.13.2              # via pygit2
+chardet==3.0.4            # via requests
+coloredlogs==10.0
+funcy==1.14               # via pyldavis
+future==0.18.2            # via pyldavis
+humanfriendly==4.18       # via coloredlogs
+idna==2.8                 # via requests
+importlib-metadata==1.4.0  # via pluggy, pytest
+jinja2==2.10.3            # via pyldavis
+joblib==0.14.1            # via pyldavis, scikit-learn
+markupsafe==1.1.1         # via jinja2
+more-itertools==8.1.0     # via pytest, zipp
+numexpr==2.7.1            # via pyldavis
+numpy==1.17.2             # via numexpr, pandas, pyldavis, scikit-learn, scipy, seriate
+ortools==7.4.7247         # via seriate
+packaging==20.1           # via pytest, seriate
+pandas==0.25.1            # via pyldavis
+pluggy==0.13.1            # via pytest
+protobuf==3.10.0          # via ortools
+py==1.8.1                 # via pytest
+pycparser==2.19           # via cffi
+pygit2==1.0.2
+pyldavis==2.1.2
+pyparsing==2.4.6          # via packaging
+pystemmer==1.3.0
+pytest==5.3.4             # via pyldavis
+python-dateutil==2.8.1    # via pandas
+pytz==2019.3              # via pandas
+requests==2.22.0
+scikit-learn==0.21.3
+scipy==1.3.1              # via pyldavis, scikit-learn
+seriate==1.1.1
+six==1.14.0               # via ortools, packaging, protobuf, python-dateutil
+tqdm==4.36.1
+urllib3==1.25.8           # via requests
+wcwidth==0.1.8            # via pytest
+wheel==0.33.6             # via pyldavis
+zipp==2.0.1               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/env-example.sh b/env-example.sh
@@ -0,0 +1 @@
+export GITHUB_TOKEN="YOUR-TOKEN-HERE"
diff --git a/requirements-tf.txt b/requirements-tf.txt
diff --git a/scripts/build-tree-sitter-library.py b/scripts/build-tree-sitter-library.py
@@ -0,0 +1,19 @@
+if __name__ == "__main__":
+    from argparse import ArgumentParser
+
+    from tree_sitter import Language
+
+    parser = ArgumentParser("Buid .so for tree-sitter")
+    parser.add_argument(
+        "--languages",
+        required=True,
+        nargs="+",
+        help="Path to the git repositories of the languages to build.",
+    )
+    parser.add_argument(
+        "--output", required=True, help="Path of the output .so to produce.",
+    )
+
+    args = parser.parse_args()
+
+    Language.build_library(args.output, args.languages)
diff --git a/scripts/install-bigartm b/scripts/install-bigartm
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        git \
+        libboost-chrono-dev \
+        libboost-date-time-dev \
+        libboost-dev \
+        libboost-filesystem-dev \
+        libboost-iostreams-dev \
+        libboost-program-options-dev \
+        libboost-system-dev \
+        libboost-thread-dev \
+        libboost-timer-dev \
+        make \
+    && pip3 install --no-cache-dir -r conf/requirements-bigartm.txt \
+    && git clone \
+        --depth 1 \
+        --branch v0.10.0 \
+        https://github.com/bigartm/bigartm.git \
+        /opt/bigartm \
+    && mkdir /opt/bigartm/build \
+    && cd /opt/bigartm/build \
+    && cmake -DINSTALL_PYTHON_PACKAGE=ON -DPYTHON=python3 .. \
+    && make -j$(getconf _NPROCESSORS_ONLN) \
+    && make install
diff --git a/scripts/install-nltk-data b/scripts/install-nltk-data
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [[ -z "${NLTK_DATA}" ]]; then
+  echo "Please set the env variable NLTK_DATA to the desired directory path."
+  exit 1
+fi
+
+apt-get update \
+    && apt-get install -y --no-install-recommends curl unzip \
+    && mkdir "$NLTK_DATA/corpora" \
+    && curl -sSL \
+        "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip" \
+        -o "$NLTK_DATA/corpora/stopwords.zip" \
+    && unzip "$NLTK_DATA/corpora/stopwords.zip" -d "$NLTK_DATA/corpora/" \
+    && rm "$NLTK_DATA/corpora/stopwords.zip" \
diff --git a/scripts/install-tree-sitter b/scripts/install-tree-sitter
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ -z "${TREE_SITTER_LANGUAGES_SO}" ]]; then
+  echo "Please set the env variable TREE_SITTER_LANGUAGES_SO to the desired output path."
+  exit 1
+fi
+
+apt-get update \
+    && apt-get install -y --no-install-recommends build-essential \
+    && pip3 install --no-cache-dir -r conf/requirements-tree-sitter.txt \
+    && git clone \
+        --branch v0.13.0 \
+        --depth 1 \
+        https://github.com/tree-sitter/tree-sitter-java \
+        /opt/tree-sitter-java \
+    && python scripts/build-tree-sitter-library.py \
+        --languages /opt/tree-sitter-java \
+        --output "$TREE_SITTER_LANGUAGES_SO"
