Merge branch 'master' of ssh://github.com/jupyterhub/binderhub into main

Author: iwilltry42

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    # At 13:29 on Friday - https://crontab.guru/
+    - cron: '29 13 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['python', 'javascript']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.travis.yml

@@ -1,25 +1,27 @@
-dist: xenial
+dist: bionic
 language: python
 python:
-  - 3.6
+  - 3.7
 git:
   depth: false
 services:
   - docker
-sudo: required
 
 before_install:
-  - nvm install 8; nvm use 8
+  - nvm install 12; nvm use 12
+  - pip install --upgrade pip setuptools
 install:
-  # need gnutls headers for pycurl
-  - sudo apt-get -yq install libgnutls28-dev
   - export PATH=$PWD/bin:$PATH
-  - pip install --upgrade pip setuptools
   - ./ci/install.sh
   - pip install --upgrade -r dev-requirements.txt
   - npm install
   - npm run webpack
   - pip install --upgrade . -r helm-chart/images/binderhub/requirements.txt
+  # NOTE: The latest docker python package (4.3.0) requires a more modern docker
+  #       version (newer than 18.06.0-ce / API version: 1.38) which is not yet
+  #       available on travis.
+  #       ref: https://github.com/jupyterhub/zero-to-jupyterhub-k8s/pull/1743
+  - pip install -U "docker~=4.2.0"
   - pip freeze
 
 script:
@@ -46,11 +48,12 @@ env:
     - TEST=auth
     - TEST=helm
   global:
-    - ASYNC_TEST_TIMEOUT=15
-    - MINIKUBE_VERSION=0.33.1
-    - KUBE_VERSION=1.13.2
-    - HELM_VERSION=2.13.0
-    - CHANGE_MINIKUBE_NONE_USER=true
+    # minikube version: https://github.com/kubernetes/minikube/releases
+    - MINIKUBE_VERSION=1.10.1
+    # kubernetes version: https://github.com/kubernetes/kubernetes/releases
+    - KUBE_VERSION=1.18.2
+    # helm version: https://github.com/helm/helm/releases
+    - HELM_VERSION=2.16.7
     - secure: BuO4oUz5YZvWhH919Tk8h3McM60NybLImIwB+0C4cmcDC/Z4uS1Jh8R8UbD3vIkJyjPgd+WuoaYGzzgJfCiS5i5TZZfi5yI0URu67fc044vaS7zSegZWsuN40mP4QNYTb+VdYniav8pqqyPyUpKNbOQ+/YJ+BWrC/ncqSL+P+UR6PE9T3TQ4XDuB45z0B2hhfDWBMpP6KtGae7YOWstIPi8ufiS76jjRzE4ziLqsOSwJRGhRbjJXqdcZeH2d54jUJSzCEGMSo5lxrFL27YOJ6Vuzr0V208AiQe60CyxtAzyiamVoE9U8pYOnv9KRDMeROSdz2HJkGetedNgCHpf0mNUWLZzQ6udzXtKeG2pxKeLDYKSm8Y3GGGa87nWRaWS1dYwCTHLe1r6Jwt6QwuBqqIa0oOMujStTEgbUOLLw5e80kSWqTxb52XnDi5SgOMGNzcylAYJLHFCL5U9ShAyZWGRAy0p7tFycXy44/k7RiqXr6Xur7NuB9bzXkmsDa7qS1t2DXoUA8FlpxfvqaSqVuFSrN16JHQeBOZGEYondQ2NPvi2vNT68Us4saWUxBf8oxF5pctbJYDDOxWpYFq/pza2+Elset9dv5KjCb61qtotLpo4PER4zgvde5/5HRa1CAOJlVfZB1Au3+b1NoVWNnPcwlb2UGut1+bcaDWyPf20=
     - secure: uqQrm6T6oGloQIHs1tLtQm3iIZFUtv1lNgtT+oCfYw3MC1CJ5AkKRaClAcVlFVOoCAE1GeCbnkob0i+LkwE+FQ1uw0fH9QyGx0PiEBrT3ilOXtkHvLXzcB/8PqPnIRR+vQnqigX2wsfiy0po99gBeaKiQjfybZ8UVp03M7+peQ4Y/Dxoj0G0nYZPsCo4F5plfdalJQX+ZVi1QlkglTotVxvb8QjrxXUrJ3OgXyVgW5fmkrRY6rWlb7RkSPlxmOhOZ7KM0BghMvox2VAD2jTatW/IEjqqaek8VqyKe1Dhw8wRUwQIwAH0VV1tUslHGcPcpbosGTPsfH7pTSUIa6ZRWd2dymMCpp7NwmFPm38QOZS/psiYNFPM6FvQnyqmtiUiU9MfcwMNJxiAUyTSslficd1Lt/aTTgBJQnK9Zd+/J6uuSc1aMGSgVNaM6eXxyNITXqz067zn8apCJm6UUEVM5FFs5SdfNXycJeV7qBwT8oe2+lvC4YzB2bHVGlnynK4IBPZsSFiBxPTBhcHVZp2FIMoC7cm/2dKW0gKPMho2Glhb7Zxn0mFej7XrT+IllD4yCjYSuSG2hpNMgE0+pc/89F2eLtey+oRAH0p4+K8DXsVCHsb7rSq+i7WDtFhFR0DBRWPG8B//CyLSbr4UoKumShbXYCI1dpSYzATmVyphQc8=
     - secure: ugOe1mIyit6BmzuTLtzVM7+edswAES1TfAnGST0EkYv341z0NnqRGdRJYtH1q1rRoZ6ciSnjHdVe8zmentxDwRJSi50888ro8EG7NnvAjG31cdwEgm7Vdv4sEbt7Dp8gYxKb5kWtt9w4IxPdMocpWPeNBmjctX+7QWwvIJZstWDjOtt8BhDufiTTHeiZR3FTNy5ykJXnFXn/WMsZoxI/a3GAgSzV4wKu0OzZYYk3GY20inJKUwxwWgyNbzQXecWlup6REd+YYrq0h4hJESyrtTQwHsijssCuy+mH8fxPpPZuF2WtHAqej0zGVKyUU/xmpCgdIHbjsheIyEE1leYn+USuoWqiHf0gjx2sYvBJP4HizJNPgj67UUNcUfGXzjegy6cDZE8iXW5clDRDkmWgx53VFlVgWV0TgdxtmOFK3IpQVOoaWc4q0e4T+/38rmUiBFnbbEep2q1wwjZojYhyaIrsEG9jOhtLW2rTADc5fdXURYqfPpubM1SCTseT43T86sRfOH3x8o9MSDnVejZ317oPfoZyqrL/FwpW45M68v/RAfvb6mY/5vM7Jw+70trgz4RhLQS55nuqnZ7NwK8ooX27YnvVnx3YisADz1Cf/A/bNixj8pVkJ7TTw/TnR/gTzat2wfsgPTWRtC8hhOyLSFnZh3J5OyeHQQMmdPspilE=

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing to BinderHub
 
 Welcome! As a [Jupyter](https://jupyter.org) project, we follow the
-[Jupyter contributor guide](https://jupyter.readthedocs.io/en/latest/contributor/content-contributor.html).
+[Jupyter contributor guide](https://jupyter.readthedocs.io/en/latest/contributing/content-contributor.html).
 
 There are several different setups for developing BinderHub, depending on which
 parts of it you want to change: the [documentation](#documentation-changes),
@@ -282,7 +282,7 @@ the [BinderHub documentation](https://binderhub.readthedocs.io/en/latest/setup-b
 
 By default, `minikube start` allocates 2GiB of main memory to the
 underlying VM, which might be too low to run the builder successfully.
-   
+
 You may run `minikube start --memory 8192` to start Minikube with a 8GiB
 VM underneath.
 

binderhub/app.py

@@ -30,6 +30,7 @@
 from .builder import BuildHandler
 from .health import HealthHandler
 from .launcher import Launcher
+from .log import log_request
 from .registry import DockerRegistry
 from .main import MainHandler, ParameterizedMainHandler, LegacyRedirectHandler
 from .repoproviders import (GitHubRepoProvider, GitRepoProvider,
@@ -306,20 +307,34 @@ def _valid_badge_base_url(self, proposal):
         config=True
     )
 
+    build_memory_request = ByteSpecification(
+        0,
+        help="""
+        Amount of memory to request when scheduling a build
+
+        0 reserves no memory.
+
+        This is used as the request for the pod that is spawned to do the building,
+        even though the pod itself will not be using that much memory
+        since the docker building is happening outside the pod.
+        However, it makes kubernetes aware of the resources being used,
+        and lets it schedule more intelligently.
+        """,
+        config=True,
+    )
     build_memory_limit = ByteSpecification(
         0,
         help="""
         Max amount of memory allocated for each image build process.
 
         0 sets no limit.
 
-        This is used as both the memory limit & request for the pod
-        that is spawned to do the building, even though the pod itself
-        will not be using that much memory since the docker building is
-        happening outside the pod. However, it makes kubernetes aware of
-        the resources being used, and lets it schedule more intelligently.
+        This is applied to the docker build itself via repo2docker,
+        though it is also applied to our pod that submits the build,
+        even though that pod will rarely consume much memory.
+        Still, it makes it easier to see the resource limits in place via kubernetes.
         """,
-        config=True
+        config=True,
     )
 
     debug = Bool(
@@ -572,43 +587,47 @@ def initialize(self, *args, **kwargs):
             with open(schema_file) as f:
                 self.event_log.register_schema(json.load(f))
 
-        self.tornado_settings.update({
-            "push_secret": self.push_secret,
-            "image_prefix": self.image_prefix,
-            "debug": self.debug,
-            'launcher': self.launcher,
-            'appendix': self.appendix,
-            "build_namespace": self.build_namespace,
-            "build_image": self.build_image,
-            'build_node_selector': self.build_node_selector,
-            'build_pool': self.build_pool,
-            "sticky_builds": self.sticky_builds,
-            'log_tail_lines': self.log_tail_lines,
-            'pod_quota': self.pod_quota,
-            'per_repo_quota': self.per_repo_quota,
-            'per_repo_quota_higher': self.per_repo_quota_higher,
-            'repo_providers': self.repo_providers,
-            'use_registry': self.use_registry,
-            'registry': registry,
-            'traitlets_config': self.config,
-            'google_analytics_code': self.google_analytics_code,
-            'google_analytics_domain': self.google_analytics_domain,
-            'about_message': self.about_message,
-            'banner_message': self.banner_message,
-            'extra_footer_scripts': self.extra_footer_scripts,
-            'jinja2_env': jinja_env,
-            'build_memory_limit': self.build_memory_limit,
-            'build_docker_host': self.build_docker_host,
-            'base_url': self.base_url,
-            'badge_base_url': self.badge_base_url,
-            "static_path": os.path.join(HERE, "static"),
-            'static_url_prefix': url_path_join(self.base_url, 'static/'),
-            'template_variables': self.template_variables,
-            'executor': self.executor,
-            'auth_enabled': self.auth_enabled,
-            'event_log': self.event_log,
-            'normalized_origin': self.normalized_origin
-        })
+        self.tornado_settings.update(
+            {
+                "log_function": log_request,
+                "push_secret": self.push_secret,
+                "image_prefix": self.image_prefix,
+                "debug": self.debug,
+                "launcher": self.launcher,
+                "appendix": self.appendix,
+                "build_namespace": self.build_namespace,
+                "build_image": self.build_image,
+                "build_node_selector": self.build_node_selector,
+                "build_pool": self.build_pool,
+                "sticky_builds": self.sticky_builds,
+                "log_tail_lines": self.log_tail_lines,
+                "pod_quota": self.pod_quota,
+                "per_repo_quota": self.per_repo_quota,
+                "per_repo_quota_higher": self.per_repo_quota_higher,
+                "repo_providers": self.repo_providers,
+                "use_registry": self.use_registry,
+                "registry": registry,
+                "traitlets_config": self.config,
+                "google_analytics_code": self.google_analytics_code,
+                "google_analytics_domain": self.google_analytics_domain,
+                "about_message": self.about_message,
+                "banner_message": self.banner_message,
+                "extra_footer_scripts": self.extra_footer_scripts,
+                "jinja2_env": jinja_env,
+                "build_memory_limit": self.build_memory_limit,
+                "build_memory_request": self.build_memory_request,
+                "build_docker_host": self.build_docker_host,
+                "base_url": self.base_url,
+                "badge_base_url": self.badge_base_url,
+                "static_path": os.path.join(HERE, "static"),
+                "static_url_prefix": url_path_join(self.base_url, "static/"),
+                "template_variables": self.template_variables,
+                "executor": self.executor,
+                "auth_enabled": self.auth_enabled,
+                "event_log": self.event_log,
+                "normalized_origin": self.normalized_origin,
+            }
+        )
         if self.auth_enabled:
             self.tornado_settings['cookie_secret'] = os.urandom(32)
 

binderhub/base.py

@@ -118,6 +118,10 @@ async def get(self):
 
 class VersionHandler(BaseHandler):
     """Serve information about versions running"""
+
+    # demote logging of 200 responses to debug-level
+    log_success_debug = True
+
     async def get(self):
         self.set_header("Content-type", "application/json")
         self.write(json.dumps(

binderhub/build.py

@@ -36,9 +36,28 @@ class Build:
         API instead of having to invent our own locking code.
 
     """
-    def __init__(self, q, api, name, namespace, repo_url, ref, git_credentials, build_image,
-                 image_name, push_secret, memory_limit, docker_host, node_selector,
-                 appendix='', log_tail_lines=100, sticky_builds=False):
+
+    def __init__(
+        self,
+        q,
+        api,
+        name,
+        *,
+        namespace,
+        repo_url,
+        ref,
+        build_image,
+        docker_host,
+        image_name,
+        git_credentials=None,
+        push_secret=None,
+        memory_limit=0,
+        memory_request=0,
+        node_selector=None,
+        appendix="",
+        log_tail_lines=100,
+        sticky_builds=False,
+    ):
         self.q = q
         self.api = api
         self.repo_url = repo_url
@@ -50,6 +69,7 @@ def __init__(self, q, api, name, namespace, repo_url, ref, git_credentials, buil
         self.build_image = build_image
         self.main_loop = IOLoop.current()
         self.memory_limit = memory_limit
+        self.memory_request = memory_request
         self.docker_host = docker_host
         self.node_selector = node_selector
         self.appendix = appendix
@@ -255,7 +275,7 @@ def submit(self):
                         volume_mounts=volume_mounts,
                         resources=client.V1ResourceRequirements(
                             limits={'memory': self.memory_limit},
-                            requests={'memory': self.memory_limit}
+                            requests={'memory': self.memory_request},
                         ),
                         env=env
                     )