Extract HttpOpener response charset from Content-Type header. (#513)

The `Accept-Charset` request header value is generally not suitable as a charset name.

Should be applied to compressed content as well.

Author: blackwinter

metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java

@@ -65,12 +65,16 @@ public final class HttpOpener extends DefaultObjectPipe<String, ObjectReceiver<R
     public static final String HEADER_FIELD_SEPARATOR = "\n";
     public static final String HEADER_VALUE_SEPARATOR = ":";
     public static final String INPUT_DESIGNATOR = "@-";
+    public static final String MIME_PARAMETER_CHARSET = "charset";
+    public static final String MIME_PARAMETER_SEPARATOR = ";";
+    public static final String MIME_PARAMETER_VALUE_SEPARATOR = "=";
 
     public static final String DEFAULT_METHOD_NAME = "GET";
     public static final Method DEFAULT_METHOD = Method.valueOf(DEFAULT_METHOD_NAME);
 
     private static final Pattern HEADER_FIELD_SEPARATOR_PATTERN = Pattern.compile(HEADER_FIELD_SEPARATOR);
     private static final Pattern HEADER_VALUE_SEPARATOR_PATTERN = Pattern.compile(HEADER_VALUE_SEPARATOR);
+    private static final Pattern MIME_PARAMETER_SEPARATOR_PATTERN = Pattern.compile(MIME_PARAMETER_SEPARATOR);
 
     private static final int ALLOWED_REDIRECTIONS = 3;
     private static final int CONNECTION_TIMEOUT = 11000;
@@ -310,7 +314,7 @@ private Reader doPostOrPut(final String requestBody, final URL urlToOpen) throws
         headers.forEach(connection::setRequestProperty);
         connection.getOutputStream().write(requestBody.getBytes());
         final InputStream inputStream = getInputStream(connection);
-        return new InputStreamReader(inputStream, headers.get(ACCEPT_CHARSET_HEADER));
+        return new InputStreamReader(inputStream, getContentCharset(connection));
     }
 
     private Reader doGet(final String requestUrl) throws IOException {
@@ -321,10 +325,10 @@ private Reader doGet(final String requestUrl) throws IOException {
 
         if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
             final GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
-            reader = new InputStreamReader(gzipInputStream);
+            reader = new InputStreamReader(gzipInputStream, getContentCharset(connection));
         }
         else {
-            reader = new InputStreamReader(inputStream, headers.get(ACCEPT_CHARSET_HEADER));
+            reader = new InputStreamReader(inputStream, getContentCharset(connection));
         }
         return reader;
     }
@@ -371,6 +375,25 @@ private InputStream getErrorStream(final InputStream errorStream) {
         }
     }
 
+    private String getContentCharset(final HttpURLConnection connection) {
+        final String contentType = connection.getContentType();
+
+        if (contentType != null) {
+            final String[] parts = MIME_PARAMETER_SEPARATOR_PATTERN.split(contentType);
+
+            for (int i = 1; i < parts.length; ++i) {
+                final String parameter = parts[i].trim();
+                final int index = parameter.indexOf(MIME_PARAMETER_VALUE_SEPARATOR);
+
+                if (index != -1 && MIME_PARAMETER_CHARSET.equalsIgnoreCase(parameter.substring(0, index))) {
+                    return parameter.substring(index + 1);
+                }
+            }
+        }
+
+        return CHARSET_DEFAULT;
+    }
+
     private HttpURLConnection followRedirects(final URL startingUrl) throws IOException {
         int times = 0;
         HttpURLConnection conn;

metafacture-io/src/test/java/org/metafacture/io/HttpOpenerTest.java

@@ -240,20 +240,35 @@ public void shouldPerformPostRequestWithContentTypeParameter() throws IOExceptio
 
     @Test
     public void shouldPerformPostRequestWithCharsetParameter() throws IOException {
+        shouldPerformPostRequestWithCharsetParameter(null);
+    }
+
+    @Test
+    public void shouldPerformPostRequestWithCharsetParameterAndContentTypeResponseHeader() throws IOException {
+        shouldPerformPostRequestWithCharsetParameter("expected:<response b[ö]dy> but was:<response b[ö]dy>");
+    }
+
+    private void shouldPerformPostRequestWithCharsetParameter(final String expectedMessage) throws IOException {
         final String charset = "ISO-8859-1";
         final String header = "Accept-Charset";
         final StringValuePattern value = WireMock.equalTo(charset);
 
+        String actualMessage;
         try {
             shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> {
                 o.setMethod(HttpOpener.Method.POST);
                 o.setUrl(u);
                 o.setAcceptCharset(charset);
-            }, s -> s.withHeader(header, value), q -> q.withHeader(header, value), null);
+            }, s -> s.withHeader(header, value), q -> q.withHeader(header, value), expectedMessage != null ?
+                r -> r.withHeader(HttpOpener.CONTENT_TYPE_HEADER, "text/plain; charset=" + charset) : null);
+
+            actualMessage = null;
         }
         catch (final ComparisonFailure e) {
-            Assert.assertEquals("expected:<response b[ö]dy> but was:<response b[ö]dy>", e.getMessage());
+            actualMessage = e.getMessage();
         }
+
+        Assert.assertEquals(expectedMessage, actualMessage);
     }
 
     @Test