Merge branch 'master' into issue/40

Author: dekobon

.github/workflows/main.yml

@@ -13,6 +13,22 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    if: github.ref != 'refs/heads/master'
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: sudo apt-get install -y wait-for-it
+      - name: Run tests - latest njs version
+        run: ./test.sh latest-njs-oss
+      - name: Run tests - stable njs version
+        run: ./test.sh oss
+
+  build_and_deploy:
+    runs-on: ubuntu-latest
+
+    if: github.ref == 'refs/heads/master'
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2

Dockerfile.oss

@@ -1,7 +1,7 @@
 FROM nginx:1.23.1
 
 ENV NGINX_VERSION "1.23.1"
-ENV NJS_VERSION   0.7.6
+ENV NJS_VERSION   0.7.7
 
 ENV PROXY_CACHE_VALID_OK "1h"
 ENV PROXY_CACHE_VALID_NOTFOUND "1m"

Dockerfile.plus

@@ -2,7 +2,7 @@ FROM debian:bullseye-slim
 
 ENV NGINX_PLUS_VERSION   27-1
 ENV NGINX_VERSION        1.21.6
-ENV NJS_VERSION          27+0.7.4-1
+ENV NJS_VERSION          27+0.7.7-1
 ENV XSLT_VERSION         27-1
 
 ENV PROXY_CACHE_VALID_OK "1h"

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -50,6 +50,22 @@ if [ "${AWS_SIGS_VERSION}" != "2" ] && [ "${AWS_SIGS_VERSION}" != "4" ]; then
   failed=1
 fi
 
+parseBoolean() {
+  case "$1" in
+    TRUE | true | True | YES | Yes | 1)
+      echo 1
+      ;;
+    *)
+      echo 0
+      ;;
+  esac
+}
+
+if [ "$(parseBoolean ${ALLOW_DIRECTORY_LIST})" == "1" ] && [ "$(parseBoolean ${PROVIDE_INDEX_PAGE})" == "1" ]; then
+  >&2 echo "ALLOW_DIRECTORY_LIST and PROVIDE_INDEX_PAGE cannot be both set"
+  failed=1
+fi
+
 if [ $failed -gt 0 ]; then
   exit 1
 fi
@@ -62,3 +78,5 @@ echo "Addressing Style: ${S3_STYLE}"
 echo "AWS Signatures Version: v${AWS_SIGS_VERSION}"
 echo "DNS Resolvers: ${DNS_RESOLVERS}"
 echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}"
+echo "Provide Index Pages Enabled: ${PROVIDE_INDEX_PAGE}"
+echo "Append slash for directory enabled: ${APPEND_SLASH_FOR_POSSIBLE_DIRECTORY}"

common/etc/nginx/include/s3gateway.js

@@ -32,9 +32,13 @@ var fs = require('fs');
  */
 var debug = _parseBoolean(process.env['S3_DEBUG']);
 var allow_listing = _parseBoolean(process.env['ALLOW_DIRECTORY_LIST'])
+var provide_index_page = _parseBoolean(process.env['PROVIDE_INDEX_PAGE'])
+var append_slash = _parseBoolean(process.env['APPEND_SLASH_FOR_POSSIBLE_DIRECTORY'])
 
 var s3_style = process.env['S3_STYLE'];
 
+var INDEX_PAGE = "index.html";
+
 /**
  * The current moment as a timestamp. This timestamp will be used across
  * functions in order for there to be no variations in signatures.
@@ -348,11 +352,14 @@ function s3uri(r) {
     if (allow_listing) {
         var queryParams = _s3DirQueryParams(uriPath, r.method);
         if (queryParams.length > 0) {
-            path = basePath + '/?' + queryParams;
+            path = basePath + '?' + queryParams;
         } else {
             path = basePath + uriPath;
         }
     } else {
+        if (provide_index_page  && _isDirectory(uriPath) ) {
+               uriPath += INDEX_PAGE;
+        }
         path = basePath + uriPath;
     }
 
@@ -374,6 +381,11 @@ function _s3DirQueryParams(uriPath, method) {
         return '';
     }
 
+    // return if static website. We don't want to list the files in the directory, we want to append the index page and get the fil.
+    if (provide_index_page){
+        return '';
+    }
+
     let path = 'delimiter=%2F'
 
     if (uriPath !== '/') {
@@ -406,13 +418,25 @@ function redirectToS3(r) {
 
     if (isDirectoryListing && r.method === 'GET') {
         r.internalRedirect("@s3Listing");
-    } else if (!isDirectoryListing && uriPath === '/') {
-        r.internalRedirect("@error404");
+    } else if ( provide_index_page == true ) {
+        r.internalRedirect("@s3");   
+    } else if ( !allow_listing && !provide_index_page && uriPath == "/" ) {
+       r.internalRedirect("@error404");
     } else {
         r.internalRedirect("@s3");
     }
 }
 
+function trailslashControl(r) {
+    if (append_slash) {
+        var hasExtension = /\/[^.\/]+\.[^.]+$/;
+        if (!hasExtension.test(r.variables.uri_path)  && !_isDirectory(r.variables.uri_path)){
+            return r.internalRedirect("@trailslash");
+        }
+    }
+        r.internalRedirect("@error404");
+}
+
 /**
  * Create HTTP Authorization header for authenticating with an AWS compatible
  * v2 API.
@@ -431,6 +455,10 @@ function signatureV2(r, bucket, credentials) {
      * nginx, then in S3 we need to request /?delimiter=/&prefix=dir1/
      * Thus, we can't put the path /dir1/ in the string to sign. */
     var uri = _isDirectory(r.variables.uri_path) ? '/' : r.variables.uri_path;
+    // To return index pages + index.html
+    if (provide_index_page && _isDirectory(r.variables.uri_path)){
+        uri = r.variables.uri_path + INDEX_PAGE
+    }
     var hmac = mod_hmac.createHmac('sha1', credentials.secretAccessKey);
     var httpDate = s3date(r);
     var stringToSign = method + '\n\n\n' + httpDate + '\n' + '/' + bucket + uri;
@@ -1045,6 +1073,7 @@ export default {
     s3auth,
     s3SecurityToken,
     s3uri,
+    trailslashControl,
     redirectToS3,
     editAmzHeaders,
     filterListResponse,

common/etc/nginx/nginx.conf

@@ -20,6 +20,8 @@ env AWS_SIGS_VERSION;
 env S3_DEBUG;
 env S3_STYLE;
 env ALLOW_DIRECTORY_LIST;
+env PROVIDE_INDEX_PAGE;
+env APPEND_SLASH_FOR_POSSIBLE_DIRECTORY;
 env PROXY_CACHE_VALID_OK;
 env PROXY_CACHE_VALID_NOTFOUND;
 env PROXY_CACHE_VALID_FORBIDDEN;

common/etc/nginx/templates/default.conf.template

@@ -112,7 +112,9 @@ server {
         proxy_intercept_errors on;
 
         # Comment out this line to receive the error messages returned by S3
-        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+        error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        error_page 404 @trailslashControl;
 
         proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3uri;
     }
@@ -169,6 +171,17 @@ server {
         return 404;
     }
 
+    location @trailslashControl {
+        # Checks if requesting a folder without trailing slash, and return 302 
+        # appending a slash to it when using for static site hosting.
+        js_content s3gateway.trailslashControl;
+    }
+
+    location @trailslash {
+        # 302 to request without slashes
+        rewrite ^ $scheme://$http_host$request_uri/ redirect;
+    }
+
     # Provide a hint to the client on 405 errors of the acceptable request methods
     error_page 405 @error405;
     location @error405 {

docs/getting_started.md

@@ -13,7 +13,9 @@
 The following environment variables are used to configure the gateway when
 running as a Container or as a Systemd service.
 
-* `ALLOW_DIRECTORY_LIST` - Enable directory listing - either true or false
+* `ALLOW_DIRECTORY_LIST` - Flag (true/false) enabling directory listing (default: false)
+* `PROVIDE_INDEX_PAGE` - Flag (true/false) which returns the index page if there is one when requesting a directory. Cannot be enabled with `ALLOW_DIRECTORY_LIST`. (default: false)
+* `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` - Flag (true/false) enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. (default: false)
 * `AWS_SIGS_VERSION` - AWS Signatures API version - either 2 or 4
 * `DNS_RESOLVERS` - (optional) DNS resolvers (separated by single spaces) to configure NGINX with
 * `S3_ACCESS_KEY_ID` - Access key
@@ -78,6 +80,16 @@ result in log messages like:
 Another limitation is that when using v2 signatures with HEAD requests, the
 gateway will not return 200 for valid folders.
 
+### Static Site Hosting
+
+When `PROVIDE_INDEX_PAGE` environment variable is set to 1, the gateway will
+transform `/some/path/` to `/some/path/index.html` when retrieving from S3.  
+Default of "index.html" can be edited in `s3gateway.js`. 
+It will also redirect `/some/path` to `/some/path/` when S3 returns 404 on 
+`/some/path` if `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` is set. `path` has to 
+look like a possible directory, it must not start with a `.` and not have an 
+extension.  
+
 ## Running as a Systemd Service
 
 A [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows

settings.example

@@ -9,6 +9,8 @@ S3_STYLE=virtual
 S3_DEBUG=false
 AWS_SIGS_VERSION=4
 ALLOW_DIRECTORY_LIST=false
+PROVIDE_INDEX_PAGE=false
+APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false
 PROXY_CACHE_VALID_OK=1h
 PROXY_CACHE_VALID_NOTFOUND=1m
 PROXY_CACHE_VALID_FORBIDDEN=30s

test.sh

@@ -102,36 +102,42 @@ if [ "${nginx_type}" = "plus" ]; then
 fi
 
 compose() {
-  ${docker_compose_cmd} -f "${test_compose_config}" -p "${test_compose_project}" "$@"
+  "${docker_compose_cmd}"  -f "${test_compose_config}" -p "${test_compose_project}" "$@"
 }
 
 integration_test() {
   printf "\033[34;1m▶\033[0m"
   printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1"
   printf "\033[34;1m▶\033[0m"
   printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4"
 
   # See if Minio is already running, if it isn't then we don't need to build it
-  if [ -z "$(docker ps -q -f name=${test_compose_project}_minio_1)" ]; then
+  # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
+
+  if [ -z "$(docker ps -q -f name=${test_compose_project}_minio-_1)" ]; then
     p "Building Docker Compose environment"
-    AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 compose up --no-start
+    COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up --no-start
 
     p "Adding test data to container"
     echo "Copying contents of ${test_dir}/data to Docker container ${test_compose_project}_minio_1:/"
-    ${docker_cmd} cp "${test_dir}/data" ${test_compose_project}_minio_1:/
+    "${docker_cmd}" cp "${test_dir}/data" "${test_compose_project}"_minio_1:/
     echo "Docker diff output:"
-    ${docker_cmd} diff ${test_compose_project}_minio_1
+    "${docker_cmd}" diff "${test_compose_project}"_minio_1
   fi
 
   p "Starting Docker Compose Environment"
-  AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 compose up -d
+  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d
 
-  if [ ${wait_for_it_installed} ]; then
+  if [ "${wait_for_it_installed}" ]; then
     # Hit minio's health check end point to see if it has started up
     for (( i=1; i<=3; i++ ))
     do
       echo "Querying minio server to see if it is ready"
-      minio_is_up="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' ${minio_server}/minio/health/cluster)"
+      minio_is_up="$(${curl_cmd} -s -o /dev/null -w '%{http_code}' "${minio_server}"/minio/health/cluster)"
       if [ "${minio_is_up}" = "200" ]; then
         break
       else
@@ -140,13 +146,13 @@ integration_test() {
     done
 
     if [ -x "${wait_for_it_cmd}" ]; then
-      $wait_for_it_cmd -h ${nginx_server_host} -p ${nginx_server_port}
+      "${wait_for_it_cmd}" -h "${nginx_server_host}" -p "${nginx_server_port}"
     fi
   fi
 
   p "Starting HTTP API tests (v$1 signatures)"
-  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2"
-  bash "${test_dir}/integration/test_api.sh" "$test_server" "$test_dir" "$1" "$2";
+  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4"
+  bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4";
 
   # We check to see if NGINX is in fact using the correct version of AWS
   # signatures as it was configured to do.
@@ -205,7 +211,9 @@ fi
 ### UNIT TESTS
 
 p "Running unit tests in Docker image"
-${docker_cmd} run \
+#MSYS_NO_PATHCONV=1 added to resolve automatic path conversion
+# https://github.com/docker/for-win/issues/6754#issuecomment-629702199
+MSYS_NO_PATHCONV=1 "${docker_cmd}" run \
   --rm \
   -v "$(pwd)/test/unit:/var/tmp" \
   --workdir /var/tmp \
@@ -225,21 +233,31 @@ ${docker_cmd} run \
 ### INTEGRATION TESTS
 
 p "Testing API with AWS Signature V2 and allow directory listing off"
-integration_test 2 0
+integration_test 2 0 0 0
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing on"
-integration_test 2 1
+integration_test 2 1 0 0
+
+compose stop nginx-s3-gateway # Restart with new config
+
+p "Testing API with AWS Signature V2 and static site on"
+integration_test 2 0 1 0
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and allow directory listing off"
-integration_test 4 0
+integration_test 4 0 0 0
+
+compose stop nginx-s3-gateway # Restart with new config
+
+p "Test API with AWS Signature V4 and allow directory listing on and appending /"
+integration_test 4 1 0 1
 
 compose stop nginx-s3-gateway # Restart with new config
 
-p "Test API with AWS Signature V4 and allow directory listing on"
-integration_test 4 1
+p "Test API with AWS Signature V4 and static site on appending /"
+integration_test 4 0 1 1
 
 p "All integration tests complete"