diff --git a/Dockerfile.buildkit.plus b/Dockerfile.buildkit.plus index 6f5d8c93..037fc447 100644 --- a/Dockerfile.buildkit.plus +++ b/Dockerfile.buildkit.plus @@ -16,6 +16,7 @@ ENV PROXY_CACHE_SLICE_SIZE=1m ENV PROXY_CACHE_VALID_OK=1h ENV PROXY_CACHE_VALID_NOTFOUND=1m ENV PROXY_CACHE_VALID_FORBIDDEN=30s +ENV PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504' # CORS env vars ENV CORS_ENABLED=0 diff --git a/Dockerfile.oss b/Dockerfile.oss index 9b24480e..9d500f3d 100644 --- a/Dockerfile.oss +++ b/Dockerfile.oss @@ -11,6 +11,7 @@ ENV PROXY_CACHE_SLICE_SIZE=1m ENV PROXY_CACHE_VALID_OK=1h ENV PROXY_CACHE_VALID_NOTFOUND=1m ENV PROXY_CACHE_VALID_FORBIDDEN=30s +ENV PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504' # CORS env vars ENV CORS_ENABLED=0 diff --git a/Dockerfile.plus b/Dockerfile.plus index a81682be..b954defc 100644 --- a/Dockerfile.plus +++ b/Dockerfile.plus @@ -16,6 +16,7 @@ ENV PROXY_CACHE_SLICE_SIZE=1m ENV PROXY_CACHE_VALID_OK=1h ENV PROXY_CACHE_VALID_NOTFOUND=1m ENV PROXY_CACHE_VALID_FORBIDDEN=30s +ENV PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504' # CORS env vars ENV CORS_ENABLED=0 diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index 4f5f9224..ec3ce234 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -38,6 +38,7 @@ env PROXY_CACHE_VALID_OK; env PROXY_CACHE_SLICE_SIZE; env PROXY_CACHE_VALID_NOTFOUND; env PROXY_CACHE_VALID_FORBIDDEN; +env PROXY_CACHE_USE_STALE; env HEADER_PREFIXES_TO_STRIP; env FOUR_O_FOUR_ON_EMPTY_BUCKET; env STRIP_LEADING_DIRECTORY_PATH; diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index 52d1e55f..19c13cc9 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -56,7 +56,7 @@ server { proxy_cache_revalidate on; proxy_cache_background_update on; proxy_cache_lock on; - proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504; + proxy_cache_use_stale ${PROXY_CACHE_USE_STALE}; proxy_cache_key "$request_method$host$uri"; # If you need to support proxying range request, refer to this article: diff --git a/deployments/s3_express/settings.s3express.example b/deployments/s3_express/settings.s3express.example index 3dcc2c6f..2206a410 100644 --- a/deployments/s3_express/settings.s3express.example +++ b/deployments/s3_express/settings.s3express.example @@ -20,3 +20,4 @@ PROXY_CACHE_INACTIVE=60m PROXY_CACHE_VALID_OK=1h PROXY_CACHE_VALID_NOTFOUND=1m PROXY_CACHE_VALID_FORBIDDEN=30s +PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504' diff --git a/docs/getting_started.md b/docs/getting_started.md index 1590d965..0f61513b 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -15,39 +15,40 @@ The following environment variables are used to configure the gateway when running as a Container or as a Systemd service. -| Name | Required? | Allowed Values | Default | Description | -| ------------------------------------- | --------- | ---------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | -| `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | -| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | -| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | -| `AWS_SESSION_TOKEN` | No | | | Session token. | -| `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | -| `S3_REGION` | Yes | | | Region associated with API | -| `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | -| `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | -| `S3_SERVER` | Yes | | | S3 host to connect to | +| Name | Required? | Allowed Values | Default | Description | +| ------------------------------------- | --------- |--------------------------------------------| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | +| `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | +| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | +| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | +| `AWS_SESSION_TOKEN` | No | | | Session token. | +| `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | +| `S3_REGION` | Yes | | | Region associated with API | +| `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | +| `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | +| `S3_SERVER` | Yes | | | S3 host to connect to | | `S3_STYLE` | Yes | `virtual-v2`, `virtual`, `path`, `default` | `default` | The S3 host/path method.

`virtual` and `virtual-v2` represent the method that uses DNS-style bucket+hostname:port. The `default` is the same as `virtual`. In the future, the `default` value will become `virtual-v2`. See [Choosing a `S3_STYLE` Setting](#user-content-choosing-a-s3_style-setting) below for details.

`path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services. See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information. | -| `S3_SERVICE` | Yes | `s3`, `s3express` | `s3` | Configures the gateway to interface with either normal S3 buckets or S3 Express One Zone | -| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | -| `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. | -| `DIRECTORY_LISTING_PATH_PREFIX` | No | | | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing) | -| `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with | -| `PROXY_CACHE_MAX_SIZE` | No | | `10g` | Limits cache size | -| `PROXY_CACHE_INACTIVE` | No | | `60m` | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness | -| `PROXY_CACHE_SLICE_SIZE` | No | | `1m` | For requests with a `Range` header included, determines the size of the chunks in which the file is fetched. Values much smaller than the requests can lead to inefficiencies due to reading and writing many files. See [below for more details](#byte-range-requests-and-caching) | -| `PROXY_CACHE_VALID_OK` | No | | `1h` | Sets caching time for response code 200 and 302 | -| `PROXY_CACHE_VALID_NOTFOUND` | No | | `1m` | Sets caching time for response code 404 | -| `PROXY_CACHE_VALID_FORBIDDEN` | No | | `30s` | Sets caching time for response code 403 | -| `PROVIDE_INDEX_PAGE` | No | `true`, `false` | `false` | Flag which returns the index page if there is one when requesting a directory. | -| `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | -| `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers from client responses. List should be specified in lower-case and a semicolon (;) should be used to as a delimiter between values. For example: x-goog-;x-something-. Headers starting with x-amz- will be stripped by default for security reasons unless explicitly added in HEADER_PREFIXES_ALLOWED. | -| `HEADER_PREFIXES_ALLOWED` | No | | | A list of allowed prefixes for HTTP headers that are returned to the client in responses. List should be specified in lower-case and a semicolon (;) should be used to as a delimiter between values. For example: x-amz-;x-something-. It is NOT recommended to return x-amz- headers for security reasons. Think carefully about what is allowed here. | -| `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | -| `CORS_ALLOWED_ORIGIN` | No | | | Value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | -| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). | -| `PREFIX_LEADING_DIRECTORY_PATH` | No | | | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed. | -| `CORS_ALLOW_PRIVATE_NETWORK_ACCESS` | No | `true`, `false` | | Flag that enables responding to the CORS OPTIONS pre-flight request header `Access-Control-Request-Private-Network` with the `Access-Control-Allow-Private-Network` header. If the value is "true", responds with "true", if "false" responds with "false". If the environment variable is blank/not set, does not respond with any header. This value is only used if CORS is enabled. See [Private Network Access: introducing preflights](https://developer.chrome.com/blog/private-network-access-preflight/) for more information about this header. | +| `S3_SERVICE` | Yes | `s3`, `s3express` | `s3` | Configures the gateway to interface with either normal S3 buckets or S3 Express One Zone | +| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | +| `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. | +| `DIRECTORY_LISTING_PATH_PREFIX` | No | | | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing) | +| `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with | +| `PROXY_CACHE_MAX_SIZE` | No | | `10g` | Limits cache size | +| `PROXY_CACHE_INACTIVE` | No | | `60m` | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness | +| `PROXY_CACHE_SLICE_SIZE` | No | | `1m` | For requests with a `Range` header included, determines the size of the chunks in which the file is fetched. Values much smaller than the requests can lead to inefficiencies due to reading and writing many files. See [below for more details](#byte-range-requests-and-caching) | +| `PROXY_CACHE_VALID_OK` | No | | `1h` | Sets caching time for response code 200 and 302 | +| `PROXY_CACHE_VALID_NOTFOUND` | No | | `1m` | Sets caching time for response code 404 | +| `PROXY_CACHE_VALID_FORBIDDEN` | No | | `30s` | Sets caching time for response code 403 | +| `PROXY_CACHE_USE_STALE` | No | | `error timeout http_500 http_502 http_503 http_504` | Sets conditions under which stale cached data can be used. | +| `PROVIDE_INDEX_PAGE` | No | `true`, `false` | `false` | Flag which returns the index page if there is one when requesting a directory. | +| `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | +| `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers from client responses. List should be specified in lower-case and a semicolon (;) should be used to as a delimiter between values. For example: x-goog-;x-something-. Headers starting with x-amz- will be stripped by default for security reasons unless explicitly added in HEADER_PREFIXES_ALLOWED. | +| `HEADER_PREFIXES_ALLOWED` | No | | | A list of allowed prefixes for HTTP headers that are returned to the client in responses. List should be specified in lower-case and a semicolon (;) should be used to as a delimiter between values. For example: x-amz-;x-something-. It is NOT recommended to return x-amz- headers for security reasons. Think carefully about what is allowed here. | +| `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | +| `CORS_ALLOWED_ORIGIN` | No | | | Value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | +| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). | +| `PREFIX_LEADING_DIRECTORY_PATH` | No | | | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed. | +| `CORS_ALLOW_PRIVATE_NETWORK_ACCESS` | No | `true`, `false` | | Flag that enables responding to the CORS OPTIONS pre-flight request header `Access-Control-Request-Private-Network` with the `Access-Control-Allow-Private-Network` header. If the value is "true", responds with "true", if "false" responds with "false". If the environment variable is blank/not set, does not respond with any header. This value is only used if CORS is enabled. See [Private Network Access: introducing preflights](https://developer.chrome.com/blog/private-network-access-preflight/) for more information about this header. | diff --git a/settings.example b/settings.example index ca71efc9..c588884e 100644 --- a/settings.example +++ b/settings.example @@ -20,4 +20,5 @@ PROXY_CACHE_INACTIVE=60m PROXY_CACHE_VALID_OK=1h PROXY_CACHE_VALID_NOTFOUND=1m PROXY_CACHE_VALID_FORBIDDEN=30s +PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504' STRIP_LEADING_DIRECTORY_PATH=/somepath \ No newline at end of file diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index 38c8a364..a200edba 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -96,6 +96,7 @@ echo "Slice of slice for byte range requests: ${PROXY_CACHE_SLICE_SIZE}" echo "Proxy Caching Time for Valid Response: ${PROXY_CACHE_VALID_OK}" echo "Proxy Caching Time for Not Found Response: ${PROXY_CACHE_VALID_NOTFOUND}" echo "Proxy Caching Time for Forbidden Response: ${PROXY_CACHE_VALID_FORBIDDEN}" +echo "Proxy Cache Using Stale: ${PROXY_CACHE_USE_STALE:-'error timeout http_500 http_502 http_503 http_504'}" echo "CORS Enabled: ${CORS_ENABLED}" echo "CORS Allow Private Network Access: ${CORS_ALLOW_PRIVATE_NETWORK_ACCESS}" diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 2faa9c8a..1c94b2e4 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -37,6 +37,7 @@ services: PROXY_CACHE_VALID_OK: "1h" PROXY_CACHE_VALID_NOTFOUND: "1m" PROXY_CACHE_VALID_FORBIDDEN: "30s" + PROXY_CACHE_USE_STALE: "error timeout http_500 http_502 http_503 http_504" minio: image: quay.io/minio/minio:RELEASE.2023-06-09T07-32-12Z