Merge branch 'master' into issue/40

Author: dekobon

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Start container with x/y arguments
+2. View logs on '....'
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Your environment**
+* Version of the repo - a specific commit or tag
+* Version of the container used (if downloaded from Docker Hub or Github)
+* S3 backend implementation you are using (AWS, Ceph, NetApp StorageGrid, etc)
+* How you are deploying Docker/Stand-alone, etc
+* NGINX type (OSS/Plus)
+* Authentication method (IAM, IAM with Fargate, IAM with K8S, AWS Credentials, etc)
+
+**Additional context**
+Add any other context about the problem here. Any log files you want to share.
+
+**Sensitive Information**
+Be sure to redact any sensitive information such as your AWS authentication keys.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

common/etc/nginx/include/listing.xsl

@@ -1,6 +1,7 @@
 <?xml version="1.0"?>
-<xsl:stylesheet version="1.1" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+<xsl:stylesheet version="1.1" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:str="http://exslt.org/strings" extension-element-prefixes="str">
     <xsl:output method="html" encoding="utf-8" indent="yes"/>
+    <xsl:strip-space elements="*" />
 
     <xsl:template match="/">
         <xsl:choose>
@@ -81,7 +82,7 @@
                           select="substring-after(text(), $globalPrefix)"/>
             <tr>
                 <td>
-                    <a href="/{text()}">
+                    <a><xsl:attribute name="href">/<xsl:call-template name="encode-uri"><xsl:with-param name="uri" select="text()"/></xsl:call-template>/</xsl:attribute>
                         <xsl:value-of select="$dirName"/>
                     </a>
                 </td>
@@ -103,7 +104,8 @@
             <xsl:variable name="size" select="*[local-name()='Size']/text()"/>
             <tr>
                 <td>
-                    <a href="/{$key}">
+                    <a>
+                        <xsl:attribute name="href">/<xsl:call-template name="encode-uri"><xsl:with-param name="uri" select="$key"/></xsl:call-template></xsl:attribute>
                         <xsl:value-of select="$fileName"/>
                     </a>
                 </td>
@@ -116,4 +118,21 @@
             </tr>
         </xsl:if>
     </xsl:template>
+    <!-- This template escapes the URI such that symbols or unicode characters are
+         encoded so that they form a valid link that NGINX can parse -->
+    <xsl:template name="encode-uri">
+        <xsl:param name="uri"/>
+        <xsl:for-each select="str:split($uri, '/')">
+            <xsl:variable name="encoded" select="str:encode-uri(., 'true', 'UTF-8')" />
+            <xsl:variable name="more-encoded" select="
+                str:replace(
+                    str:replace(
+                        str:replace(
+                            str:replace(
+                                str:replace($encoded, '@', '%40'), '(', '%28'),
+                        ')', '%29'),
+                    '!', '%21'),
+                '*', '%2A')" />
+            <xsl:value-of select="$more-encoded" /><xsl:if test="position() != last()">/</xsl:if></xsl:for-each>
+    </xsl:template>
 </xsl:stylesheet>

common/etc/nginx/include/s3gateway.js

@@ -354,13 +354,14 @@ function s3uri(r) {
         if (queryParams.length > 0) {
             path = basePath + '?' + queryParams;
         } else {
-            path = basePath + uriPath;
+            path = _escapeURIPath(basePath + uriPath);
         }
     } else {
+        // This is a path that will resolve to an index page
         if (provide_index_page  && _isDirectory(uriPath) ) {
-               uriPath += INDEX_PAGE;
+            uriPath += INDEX_PAGE;
         }
-        path = basePath + uriPath;
+        path = _escapeURIPath(basePath + uriPath);
     }
 
     _debug_log(r, 'S3 Request URI: ' + r.method + ' ' + path);
@@ -392,7 +393,7 @@ function _s3DirQueryParams(uriPath, method) {
         let decodedUriPath = decodeURIComponent(uriPath);
         let without_leading_slash = decodedUriPath.charAt(0) === '/' ?
             decodedUriPath.substring(1, decodedUriPath.length) : decodedUriPath;
-        path += '&prefix=' + encodeURIComponent(without_leading_slash);
+        path += '&prefix=' + _encodeURIComponent(without_leading_slash);
     }
 
     return path;
@@ -419,7 +420,7 @@ function redirectToS3(r) {
     if (isDirectoryListing && r.method === 'GET') {
         r.internalRedirect("@s3Listing");
     } else if ( provide_index_page == true ) {
-        r.internalRedirect("@s3");   
+        r.internalRedirect("@s3");
     } else if ( !allow_listing && !provide_index_page && uriPath == "/" ) {
        r.internalRedirect("@error404");
     } else {
@@ -569,8 +570,9 @@ function _buildSignatureV4(r, amzDatetime, eightDigitDate, creds, bucket, region
             uri = '/';
         }
     } else {
-        uri = _escapeURIPath(s3uri(r));
+        uri = s3uri(r);
     }
+
     var canonicalRequest = _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, creds.sessionToken);
 
     _debug_log(r, 'AWS v4 Auth Canonical Request: [' + canonicalRequest + ']');
@@ -781,6 +783,19 @@ function _padWithLeadingZeros(num, size) {
     return s.substr(s.length-size);
 }
 
+/**
+ * Adds additional encoding to a URI component
+ *
+ * @param string {string} string to encode
+ * @returns {string} an encoded string
+ * @private
+ */
+function _encodeURIComponent(string) {
+    return encodeURIComponent(string)
+        .replace(/[!*'()]/g, (c) =>
+            `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+
 /**
  * Escapes the path portion of a URI without escaping the path separator
  * characters (/).
@@ -795,7 +810,7 @@ function _escapeURIPath(uri) {
     let components = [];
 
     decodedUri.split('/').forEach(function (item, i) {
-        components[i] = encodeURIComponent(item);
+        components[i] = _encodeURIComponent(item);
     });
 
     return components.join('/');
@@ -1105,6 +1120,7 @@ export default {
     // These functions do not need to be exposed, but they are exposed so that
     // unit tests can run against them.
     _padWithLeadingZeros,
+    _encodeURIComponent,
     _eightDigitDate,
     _amzDatetime,
     _splitCachedValues,

docs/development.md

@@ -6,9 +6,9 @@ In the [examples/ directory](/examples), there are `Dockerfile` examples that
 show how to extend the base functionality of the NGINX S3 Gateway by adding
 additional modules.
 
-* [Enabling Brotli Compression in Docker](examples/brotli-compression)
-* [Enabling GZip Compression in Docker](examples/gzip-compression)
-* [Installing Modsecurity in Docker](examples/modsecurity)
+* [Enabling Brotli Compression in Docker](/examples/brotli-compression)
+* [Enabling GZip Compression in Docker](/examples/gzip-compression)
+* [Installing Modsecurity in Docker](/examples/modsecurity)
 
 ## Testing
 

test/data/bucket-1/a/%@!*()=$#^&|.txt

@@ -0,0 +1 @@
+We are but selling water next to a river.

test/data/bucket-1/a/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.txt

@@ -0,0 +1,4 @@
+"Where the is not one thing, what then?"
+"Throw it away!"
+"With not one thing, what there is to throw away?"
+"Then carry it off!"

test/data/bucket-1/a/plus+plus.txt

@@ -0,0 +1,27 @@
+代悲白頭翁　　　Lament for the White-Haired Old Man
+洛陽城東桃李花　In the east of Luoyang City, Peach blossoms abound
+飛來飛去落誰家　Their petals float around, coming and going, to whose house will they fall?
+洛陽女児惜顔色　Girls in Luoyang cherish their complexion
+行逢落花長歎息　They breathe a deep sigh upon seeing the petals fall
+今年花落顔色改　This year the petals fall and their complexion changes
+明年花開復誰在　Who will be there when the flowers bloom next year?
+已見松柏摧為薪　I've seen the pines and cypresses destroyed and turned into firewood
+更聞桑田変成海　I hear that the mulberry fields have fallen into the sea
+古人無復洛城東　The people of old never came back to the east of Luoyang City
+今人還對落花風　The people of today likewise face the falling flowers in the wind
+年年歳歳花相似　Year after year, flowers look alike
+歳歳年年人不同　Year after year, the people are not the same
+寄言全盛紅顔子　I want you to get this message, my child, you are in your prime, with a rosy complexion
+應憐半死白頭翁　Take pity on the half-dead white-haired old man
+此翁白頭真可憐　You really must take pity on this white-haired old man
+伊昔紅顔美少年　For once upon a time, I used to be a red-faced handsome young man
+公子王孫芳樹下　A child of noble birth under a fragrant tree
+清歌妙舞落花前　Singing and dancing in front of the falling petals
+光禄池臺開錦繍　At the platform before the mirror pond, beautiful autumn leaves opening all around
+将軍楼閣畫神仙　The general’s pavilion is painted with gods and goddesses
+一朝臥病無相識　Once I was sick and no one knew me
+三春行楽在誰邉　Who will be at the shore for the spring outing?
+宛轉蛾眉能幾時　For how long will the moths gracefully turn about?
+須臾鶴髪亂如絲　The crane’s feathers are like tangled threads for just a moment
+但看古来歌舞地　Yet, look at the ancient places of song and dance
+惟有黄昏鳥雀悲　Only in twilight, do the birds lament

test/data/bucket-1/a/これは This is ASCII системы חן .txt

@@ -0,0 +1 @@
+What name do you use to call out to the emptiness?

test/data/bucket-1/b/c/'(1).txt

@@ -0,0 +1 @@
+In the midst of movement and chaos, keep stillness inside of you.