Upgrade to deephaven 0.17.0 (#101)

* Upgrade to deephaven 0.17.0

* Locked the ib api version.

* Build ib whls.

* Use CI built IB wheels in docker builds.

* Debugging IB wheel generation

* Debugging IB wheel generation

* Use IB wheels for sphinx.

* Fix IB wheel artificat publication.

* Buid AMD64 and ARM64 docker images.

* upgrade docker publish task.

* Buid AMD64 and ARM64 docker images.

* Use IB wheels for sphinx.

* Store sphinx hotspots.

* Cleaned up launch options and related documentation.

* Don't start sphinx server.

Author: chipkent

.github/workflows/Dockerfile.pip

@@ -1,26 +1,17 @@
 
 FROM ubuntu:22.04
 
-ARG IB_VERSION
-
 RUN apt update && \
-    apt install -y openjdk-11-jdk python3-pip curl zip && \
+    apt install -y openjdk-11-jdk python3-pip python3-venv curl zip && \
     pip3 install --upgrade pip setuptools wheel
 
 ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
 
-RUN mkdir /build && \
-    cd /build && \
-    curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" && \
-    unzip api.zip && \
-    cd ./IBJts/source/pythonclient && \
-    python3 setup.py install && \
-    cd / && \
-    rm -rf /build
-
 COPY ./wheels /wheels
+COPY ./ib-wheels /ib-wheels
 
-RUN pip3 install /wheels/*.whl && \
-    rm -rf /wheels/
+RUN pip3 install /ib-wheels/*.whl && \
+    pip3 install /wheels/*.whl && \
+    rm -rf /ib-wheels/ /wheels/
 
 CMD python3

.github/workflows/Dockerfile.server

@@ -2,22 +2,13 @@ ARG DH_VERSION
 
 FROM ghcr.io/deephaven/server:${DH_VERSION}
 
-ARG IB_VERSION
-
 RUN apt update && \
     apt install -y python3-pip python3-venv curl zip && \
     pip3 install --upgrade pip setuptools wheel
 
-RUN mkdir /build && \
-    cd /build && \
-    curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" && \
-    unzip api.zip && \
-    cd ./IBJts/source/pythonclient && \
-    python3 setup.py install && \
-    cd / && \
-    rm -rf /build
-
 COPY ./wheels /wheels
+COPY ./ib-wheels /ib-wheels
 
-RUN pip3 install /wheels/*.whl && \
-    rm -rf /wheels/
+RUN pip3 install /ib-wheels/*.whl && \
+    pip3 install /wheels/*.whl && \
+    rm -rf /ib-wheels/ /wheels/

.github/workflows/build-and-publish.yml

@@ -13,9 +13,35 @@ on:
 
 env:
   IB_VERSION: 1016.01
-  DH_VERSION: 0.16.0
+  DH_VERSION: 0.17.0
 
 jobs:
+  build-ib-whl:
+    name: Build IB WHL
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools wheel build twine
+    - name: Build
+      run: |
+        curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip"
+        unzip api.zip
+        cd ./IBJts/source/pythonclient
+        python -m build
+    - name: Archive build artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: ib-wheels
+        path: |
+          ./IBJts/source/pythonclient/dist/*
+
+
   build-whl:
     name: Build WHL
     runs-on: ubuntu-20.04
@@ -74,7 +100,7 @@ jobs:
   build-sphinx:
     name: Build Sphinx
     runs-on: ubuntu-20.04
-    needs: [build-whl]
+    needs: [build-ib-whl, build-whl]
     steps:
     - uses: actions/checkout@v1
     - name: Apt installs
@@ -83,6 +109,12 @@ jobs:
         sudo apt install -y openjdk-11-jdk
     - name: Pip installs
       run:  pip3 install --upgrade sphinx==4.2.0 sphinx-autodoc-typehints furo==2021.10.9
+    - name: Download IB wheels
+      uses: actions/download-artifact@v3
+      with:
+        name: ib-wheels
+    - name: Install IB Whl
+      run: pip3 install *.whl
     - name: Download wheels
       uses: actions/download-artifact@v3
       with:
@@ -109,6 +141,7 @@ jobs:
         path: '**/*_pid*.log'
         if-no-files-found: ignore
 
+
   publish-sphinx:
     name: Publish Sphinx
     runs-on: ubuntu-20.04
@@ -126,11 +159,19 @@ jobs:
       with:
         branch: gh-pages
         folder: html/
+    - name: Upload JVM Error Logs
+      uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: sphinx-jvm-err
+        path: '**/*_pid*.log'
+        if-no-files-found: ignore
+
 
   docker-pip:
     name: Build and Publish Docker (pip-installed Deephaven)
     runs-on: ubuntu-20.04
-    needs: [build-whl]
+    needs: [build-ib-whl, build-whl]
     permissions:
       contents: read
       packages: write
@@ -140,6 +181,11 @@ jobs:
       - name: Get branch name
         id: branch-name
         uses: tj-actions/[email protected]
+      - name: Download IB wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: ib-wheels
+          path: ib-wheels/
       - name: Download wheels
         uses: actions/download-artifact@v3
         with:
@@ -162,10 +208,15 @@ jobs:
             type=ref,event=tag
             type=ref,event=pr
             type=raw,${{ steps.branch-name.outputs.current_branch }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2.7.0
+        uses: docker/build-push-action@v3
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           file: ./.github/workflows/Dockerfile.pip
           push: true
           tags: ${{ steps.meta-base.outputs.tags }}
@@ -174,10 +225,11 @@ jobs:
         env:
           IMAGE_NAME: ${{ github.repository }}
 
+
   docker-dhserver:
     name: Build and Publish Docker (Deephaven server image)
     runs-on: ubuntu-20.04
-    needs: [build-whl]
+    needs: [build-ib-whl, build-whl]
     permissions:
       contents: read
       packages: write
@@ -187,6 +239,11 @@ jobs:
       - name: Get branch name
         id: branch-name
         uses: tj-actions/[email protected]
+      - name: Download IB wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: ib-wheels
+          path: ib-wheels/
       - name: Download wheels
         uses: actions/download-artifact@v3
         with:
@@ -210,10 +267,15 @@ jobs:
             type=ref,event=tag
             type=ref,event=pr
             type=raw,${{ steps.branch-name.outputs.current_branch }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2.7.0
+        uses: docker/build-push-action@v3
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           file: ./.github/workflows/Dockerfile.server
           push: true
           tags: ${{ steps.meta-base.outputs.tags }}

README.md

@@ -171,42 +171,10 @@ upper right corner.  ![](https://raw.githubusercontent.com/deephaven-examples/de
 There are multiple ways to launch [Deephaven](https://deephaven.io) with [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) 
 installed.  The launch can either happen via a local installation or via Docker images.
 
-Lightweight installations use pip-installed Deephaven.  These systems have less overhead, but they lack
-a few Deephaven IDE features.  Heavyweight installations launch the entire [Deephaven](https://deephaven.io) system.
-These systems launch multiple Docker containers and have more overhead, but they provide all Deephaven IDE features.
+For a Docker Compose example, see [docker/release/](docker/release/).
 
-If you are running scripts that do not need human interaction, consider using a pip-installed option.  
-If you are interactively working with data, consider the full deephaven system.
 
-### (Option 1) Launch the full Deephaven system -- interactive
-
-> **_NOTE:_**  Deephaven does not yet have published Docker images for all architectures.  This launch should work on Linux (AMD64 and ARM64), Mac (Intel), and Windows WSL.  It is not yet supported on Windows without WSL or Mac (M1 and M2). In these cases, the `full_web_1` Docker image will exit.  This can be seen using `docker ps` or `docker compose ps`.  For these architectures, you will need to build Deephaven Docker images locally.  See [Build and launch from source](https://deephaven.io/core/docs/how-to-guides/launch-build/).
-
-
-The full Deephaven system contains the most full-featured IDE.  Running the full Deephaven system launches multiple Docker
-containers, so it has the most overhead.  This is a good option for interactively working with data.
-
-See [./docker/full](./docker/full) for more details.
-
-1) Clone the deephaven-ib repository:
-    * On Mac:
-    ```bash
-    git clone [email protected]:deephaven-examples/deephaven-ib.git
-    cd ./docker/full/
-    ```
-2) Create a directory for your data and scripts
-    ```bash
-    mkdir data
-    cp <your_data_and_scripts> data
-    ```
-3) Launch the system:
-    ```bash
-    ./run_system.sh
-    ```
-4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
-
-
-### (Option 2) Launch pip-installed Deephaven with Docker -- interactive
+### (Option 1) Launch pip-installed Deephaven with Docker -- interactive
 
 The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip.  In this case,
 the pip-installed Deephaven system is installed in a Docker container.
@@ -215,23 +183,19 @@ the pip-installed Deephaven system is installed in a Docker container.
     ```bash
     mkdir data
     ```
-2) Launch the system:
-    * On Mac:
-    ```bash
-    git clone [email protected]:deephaven-examples/deephaven-ib.git
-    ./deephaven-ib/docker/dev/build.sh
-    # Set jvm_args to the desired JVM memory for Deephaven
-    docker run -it -v data:/data -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
-    ```
-    * On other platforms:
+2) Create a directory for your Deephaven IDE configuration and notebooks
+   ```bash
+   mkdir `pwd`/.deephaven
+   ```
+3) Launch the system:
     ```bash
     # Set jvm_args to the desired JVM memory for Deephaven
-    docker run -it -v data:/data -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
+    docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
     ```
-3) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
+4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
 
 
-### (Option 3) Launch pip-installed Deephaven with Docker -- run a script
+### (Option 2) Launch pip-installed Deephaven with Docker -- run a script
 
 The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip.  In this case,
 the pip-installed Deephaven system is installed in a Docker container.  
@@ -244,23 +208,19 @@ This is a good option for production scenarios where scripts need to be run and
     # Set jvm_args to the desired JVM memory for Deephaven
     cp path/to/your_script.py data/your_script.py
     ```
-2) Launch the system and execute a custom script:
-    * On Mac:
-    ```bash
-    git clone [email protected]:deephaven-examples/deephaven-ib.git
-    ./deephaven-ib/docker/dev/build.sh
-    docker run -it -v data:/data -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i /data/your_script.py
-    ```
-    * On other platforms:
+2) Create a directory for your Deephaven IDE configuration and notebooks
+   ```bash
+   mkdir `pwd`/.deephaven
+   ```
+3) Launch the system:
     ```bash
-    docker run -it -v data:/data -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i /data/your_script.py
+    # Set jvm_args to the desired JVM memory for Deephaven
+    docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i /data/your_script.py
     ```
-3) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
-
+4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
 
-### (Option 4) Launch pip-installed Deephaven with a local installation (No Docker) -- interactive
 
-> **_NOTE:_**  Deephaven pip install is not yet supported on all architectures.  This launch should work on Linux (AMD64 and ARM64), Mac (M1 and Intel), and Windows WSL.  It is not yet supported on Windows without WSL.  For Windows, use WSL.
+### (Option 3) Launch pip-installed Deephaven with a local installation (No Docker) -- interactive
 
 The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip.  In this case,
 the pip-installed Deephaven system is installed directly on your local system, without Docker.
@@ -285,14 +245,13 @@ new feature and has not been well tested.  To do this:
 4) Launch the system:
     ```bash
     # Set jvm_args to the desired JVM memory for Deephaven
-    python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
+    # Deephaven IDE configuration and notebooks are stored to ~/.deephaven
+    python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()"
     ```
 5) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser.
 6) Use `host="localhost"` for the hostname in the examples (Windows WSL uses `host="host.docker.internal"`, since WSL is built on Docker.)
 
-### (Option 5) Launch pip-installed Deephaven with a local installation (No Docker) -- run a script
-
-> **_NOTE:_**  Deephaven pip install is not yet supported on all architectures.  This launch should work on Linux (AMD64 and ARM64), Mac (M1 and Intel), and Windows WSL.  It is not yet supported on Windows without WSL.  For Windows, use WSL.
+### (Option 4) Launch pip-installed Deephaven with a local installation (No Docker) -- run a script
 
 The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip.  In this case,
 the pip-installed Deephaven system is installed directly on your local system, without Docker.  
@@ -317,7 +276,8 @@ new feature and has not been well tested.  To do this:
     ```
 4) Launch the system and execute a custom script:
     ```bash
-    # your_script.py must begin with: "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
+    # your_script.py must begin with: "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()"
+    # Deephaven IDE configuration and notebooks are stored to ~/.deephaven
     # Set jvm_args to the desired JVM memory for Deephaven
     python3 -i your_script.py
     ```

docker/dev/README.md

@@ -4,6 +4,8 @@ This directory contains the ingredients to build Docker images from the current,
 
 This is useful when doing local development or when official images are not available for your platform.
 
+In general, you will want to use the officially released images at [https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib](https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib).
+
 ## Build Image
 
 ```bash
@@ -12,8 +14,14 @@ This is useful when doing local development or when official images are not avai
 
 ## Run image in interactive mode
 
+```bash
+docker compose up
+```
+
+or
+
 ```bash
 # Set jvm_args to the desired JVM memory for Deephaven
-docker run -it -v data:/data -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
+docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
 ```
 

docker/dev/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.4"
+
+services:
+  server:
+    image: deephaven-examples/deephaven-ib:dev
+    command: python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()"
+    stdin_open: true
+    tty: true
+    ports:
+      - "${DEEPHAVEN_PORT:-10000}:10000"
+    volumes:
+      - ./data:/data
+      - ~/.deephaven:/storage
+