Add drop auth headers redirect strategy for Azure

 - Drop Authorization headers only for Basic auth.
 - Add support for GitHub Container registry.
    - Jackson MessageConverter support for text/plain. The Github CR response's media-type is always text/plain although the content is in JSON
 - Normalize Azure redirect URI
 - Add Container Registries IT tests
 - Add SCDF image build configuration
 - Fix docker compose debug conf for build pack images

 Resolves #4412

Author: tzolov

spring-cloud-dataflow-container-registry/src/main/java/org/springframework/cloud/dataflow/container/registry/ContainerImageRestTemplateFactory.java

@@ -20,7 +20,6 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -36,7 +35,7 @@
 import org.apache.http.impl.client.HttpClients;
 
 import org.springframework.boot.web.client.RestTemplateBuilder;
-import org.springframework.cloud.dataflow.container.registry.authorization.DropAuthorizationHeaderOnSignedS3RequestRedirectStrategy;
+import org.springframework.cloud.dataflow.container.registry.authorization.DropAuthorizationHeaderRequestRedirectStrategy;
 import org.springframework.http.MediaType;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
 import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
@@ -190,14 +189,19 @@ private RestTemplate initRestTemplate(HttpClientBuilder clientBuilder, boolean w
 		HttpComponentsClientHttpRequestFactory customRequestFactory =
 				new HttpComponentsClientHttpRequestFactory(
 						clientBuilder
-								.setRedirectStrategy(new DropAuthorizationHeaderOnSignedS3RequestRedirectStrategy())
+								.setRedirectStrategy(new DropAuthorizationHeaderRequestRedirectStrategy())
+								// Azure redirects may contain double slashes and on default those are normilised
+								.setDefaultRequestConfig(RequestConfig.custom().setNormalizeUri(false).build())
 								.build());
 
 		// DockerHub response's media-type is application/octet-stream although the content is in JSON.
-		// Therefore extend the MappingJackson2HttpMessageConverter media-types to include application/octet-stream.
+		// Similarly the Github CR response's media-type is always text/plain although the content is in JSON.
+		// Therefore we extend the MappingJackson2HttpMessageConverter media-types to
+		// include application/octet-stream and text/plain.
 		MappingJackson2HttpMessageConverter octetSupportJsonConverter = new MappingJackson2HttpMessageConverter();
-		List<MediaType> mediaTypeList = new ArrayList(octetSupportJsonConverter.getSupportedMediaTypes());
+		ArrayList<MediaType> mediaTypeList = new ArrayList(octetSupportJsonConverter.getSupportedMediaTypes());
 		mediaTypeList.add(MediaType.APPLICATION_OCTET_STREAM);
+		mediaTypeList.add(MediaType.TEXT_PLAIN);
 		octetSupportJsonConverter.setSupportedMediaTypes(mediaTypeList);
 
 		return restTemplateBuilder

spring-cloud-dataflow-container-registry/src/main/java/org/springframework/cloud/dataflow/container/registry/authorization/DropAuthorizationHeaderOnSignedS3RequestRedirectStrategy.java renamed to spring-cloud-dataflow-container-registry/src/main/java/org/springframework/cloud/dataflow/container/registry/authorization/DropAuthorizationHeaderRequestRedirectStrategy.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2020 the original author or authors.
+ * Copyright 2020-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,33 +32,48 @@
 import org.apache.http.protocol.HttpContext;
 
 /**
- * The Amazon S3 API supports two Authentication Methods (https://amzn.to/2Dg9sga):
- * (1) HTTP Authorization header and (2) Query string parameters (often referred to as a pre-signed URL).
+ * Both Amazon and Azure Container Registry services require special treatment for the Authorization headers when the
+ * HTTP request are forwarded to 3rd party services.
  *
- * But only one auth mechanism is allowed at a time. If the http request contains both an Authorization header and
- * an pre-signed URL parameters then an error is thrown.
+ * Amazon:
+ *   The Amazon S3 API supports two Authentication Methods (https://amzn.to/2Dg9sga):
+ *   (1) HTTP Authorization header and (2) Query string parameters (often referred to as a pre-signed URL).
  *
- * Container Registries often use AmazonS3 as a backend object store. If HTTP Authorization header
- *  is used to authenticate with the Container Registry and then this registry redirect the request to a S3 storage
- *  using pre-signed URL authentication, the redirection will fail.
+ *   But only one auth mechanism is allowed at a time. If the http request contains both an Authorization header and
+ *   an pre-signed URL parameters then an error is thrown.
  *
- * Solution is to implement a HTTP redirect strategy that removes the original Authorization headers when the request is
- * redirected toward an Amazon signed URL.
+ *   Container Registries often use AmazonS3 as a backend object store. If HTTP Authorization header
+ *   is used to authenticate with the Container Registry and then this registry redirect the request to a S3 storage
+ *   using pre-signed URL authentication, the redirection will fail.
+ *
+ *   Solution is to implement a HTTP redirect strategy that removes the original Authorization headers when the request is
+ *   redirected toward an Amazon signed URL.
+ *
+ * Azure:
+ *   Azure have same type of issues as S3 so header needs to be dropped as well.
+ *   (https://docs.microsoft.com/en-us/azure/container-registry/container-registry-faq#authentication-information-is-not-given-in-the-correct-format-on-direct-rest-api-calls)
  *
  * @author Adam J. Weigold
+ * @author Janne Valkealahti
+ * @author Christian Tzolov
  */
-public class DropAuthorizationHeaderOnSignedS3RequestRedirectStrategy extends DefaultRedirectStrategy {
+public class DropAuthorizationHeaderRequestRedirectStrategy extends DefaultRedirectStrategy {
 
 	private static final String AMZ_CREDENTIAL = "X-Amz-Credential";
 
 	private static final String AUTHORIZATION_HEADER = "Authorization";
 
+	private static final String AZURECR_URI_SUFFIX = "azurecr.io";
+
+	private static final String BASIC_AUTH = "Basic";
+
 	@Override
 	public HttpUriRequest getRedirect(final HttpRequest request, final HttpResponse response,
 			final HttpContext context) throws ProtocolException {
 
 		HttpUriRequest httpUriRequest = super.getRedirect(request, response, context);
 
+		// Handle Amazon requests
 		final String query = httpUriRequest.getURI().getQuery();
 
 		if (StringUtils.isNoneEmpty(query) && query.contains(AMZ_CREDENTIAL)) {
@@ -69,6 +84,22 @@ public HttpUriRequest getRedirect(final HttpRequest request, final HttpResponse
 			}
 		}
 
+		// Handle Azure requests
+		if (request.getRequestLine().getUri().contains(AZURECR_URI_SUFFIX)) {
+			final String method = request.getRequestLine().getMethod();
+			if (StringUtils.isNoneEmpty(method)
+					&& (method.equalsIgnoreCase(HttpHead.METHOD_NAME) || method.equalsIgnoreCase(HttpGet.METHOD_NAME))) {
+				return new DropAuthorizationHeaderHttpRequestBase(httpUriRequest.getURI(), method) {
+					// drop headers only for the Basic Auth and leve  them unchanged for OAuth2!
+					@Override
+					protected boolean isDropHeader(String name, String value) {
+						return name.equalsIgnoreCase(AUTHORIZATION_HEADER) && StringUtils.isNoneEmpty(value)
+								&& value.contains(BASIC_AUTH);
+					}
+				};
+			}
+		}
+
 		return httpUriRequest;
 	}
 
@@ -92,38 +123,46 @@ public String getMethod() {
 
 		@Override
 		public void addHeader(Header header) {
-			if (!header.getName().equalsIgnoreCase(AUTHORIZATION_HEADER)) {
+			if (!isDropHeader(header)) {
 				super.addHeader(header);
 			}
 		}
 
 		@Override
 		public void addHeader(String name, String value) {
-			if (!name.equalsIgnoreCase(AUTHORIZATION_HEADER)) {
+			if (!isDropHeader(name, value)) {
 				super.addHeader(name, value);
 			}
 		}
 
 		@Override
 		public void setHeader(Header header) {
-			if (!header.getName().equalsIgnoreCase(AUTHORIZATION_HEADER)) {
+			if (!isDropHeader(header)) {
 				super.setHeader(header);
 			}
 		}
 
 		@Override
 		public void setHeader(String name, String value) {
-			if (!name.equalsIgnoreCase(AUTHORIZATION_HEADER)) {
+			if (!isDropHeader(name, value)) {
 				super.setHeader(name, value);
 			}
 		}
 
 		@Override
 		public void setHeaders(Header[] headers) {
 			Header[] filteredHeaders = Arrays.stream(headers)
-					.filter(header -> !header.getName().equalsIgnoreCase(AUTHORIZATION_HEADER))
+					.filter(header -> !isDropHeader(header))
 					.toArray(Header[]::new);
 			super.setHeaders(filteredHeaders);
 		}
+
+		protected boolean isDropHeader(Header header) {
+			return isDropHeader(header.getName(), header.getValue());
+		}
+
+		protected boolean isDropHeader(String name, String value) {
+			return name.equalsIgnoreCase(AUTHORIZATION_HEADER);
+		}
 	}
 }

spring-cloud-dataflow-server/pom.xml

@@ -165,6 +165,22 @@
 					</execution>
 				</executions>
 			</plugin>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>build-image</goal>
+						</goals>
+					</execution>
+				</executions>
+				<configuration>
+					<imageName>
+						springcloud/${project.artifactId}:${project.version}
+					</imageName>
+				</configuration>
+			</plugin>
 		</plugins>
 	</build>
 

spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java

@@ -288,6 +288,41 @@ public void applicationMetadataDockerTests() {
 		dataFlowOperations.appRegistryOperations().unregister("docker-app-with-jar-metadata", ApplicationType.sink);
 	}
 
+	@Test
+	@EnabledIfSystemProperty(named = "SCDF_CR_TEST", matches="true")
+	public void githubContainerRegistryTests() {
+		containerRegistryTests("github-log-sink",
+				"docker:ghcr.io/tzolov/log-sink-rabbit:3.1.0-SNAPSHOT");
+	}
+
+	@Test
+	@EnabledIfSystemProperty(named = "SCDF_CR_TEST", matches="true")
+	public void azureContainerRegistryTests() {
+		containerRegistryTests("azure-log-sink",
+				"docker:scdftest.azurecr.io/springcloudstream/log-sink-rabbit:3.1.0-SNAPSHOT");
+	}
+
+	@Test
+	@EnabledIfSystemProperty(named = "SCDF_CR_TEST", matches="true")
+	public void harborContainerRegistryTests() {
+		containerRegistryTests("harbor-log-sink",
+				"docker:projects.registry.vmware.com/scdf/scdftest/log-sink-rabbit:3.1.0-SNAPSHOT");
+	}
+
+	private void containerRegistryTests(String appName, String appUrl) {
+		logger.info("application-metadata-" + appName + "-container-registry-test");
+
+		// Docker app with container image metadata
+		dataFlowOperations.appRegistryOperations().register( appName, ApplicationType.sink,
+				appUrl, null, true);
+		DetailedAppRegistrationResource dockerAppWithContainerMetadata = dataFlowOperations.appRegistryOperations()
+				.info(appName, ApplicationType.sink, false);
+		assertThat(dockerAppWithContainerMetadata.getOptions()).hasSize(3);
+
+		// unregister the test apps
+		dataFlowOperations.appRegistryOperations().unregister(appName, ApplicationType.sink);
+	}
+
 	// -----------------------------------------------------------------------
 	//                          PLATFORM  TESTS
 	// -----------------------------------------------------------------------

spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/util/DockerComposeFactory.java

@@ -110,6 +110,12 @@ public class DockerComposeFactory {
 			.withAdditionalEnvironmentVariable("METADATA_DEFAULT_DOCKERHUB_USER", DockerComposeFactoryProperties.get("METADATA_DEFAULT_DOCKERHUB_USER", ""))
 			.withAdditionalEnvironmentVariable("METADATA_DEFAULT_DOCKERHUB_PASSWORD", DockerComposeFactoryProperties.get("METADATA_DEFAULT_DOCKERHUB_PASSWORD", ""))
 			.withAdditionalEnvironmentVariable("COMPOSE_PROJECT_NAME", "scdf")
+			.withAdditionalEnvironmentVariable("CR_AZURE_USER", DockerComposeFactoryProperties.get("CR_AZURE_USER", ""))
+			.withAdditionalEnvironmentVariable("CR_AZURE_PASS", DockerComposeFactoryProperties.get("CR_AZURE_PASS", ""))
+			.withAdditionalEnvironmentVariable("CR_GITHUB_USER", DockerComposeFactoryProperties.get("CR_GITHUB_USER", ""))
+			.withAdditionalEnvironmentVariable("CR_GITHUB_PASS", DockerComposeFactoryProperties.get("CR_GITHUB_PASS", ""))
+			.withAdditionalEnvironmentVariable("CR_HARBOR_USER", DockerComposeFactoryProperties.get("CR_HARBOR_USER", ""))
+			.withAdditionalEnvironmentVariable("CR_HARBOR_PASS", DockerComposeFactoryProperties.get("CR_HARBOR_PASS", ""))
 			.build();
 
 	private static String[] addDockerComposeToPath(String[] dockerComposePaths, String additionalDockerCompose) {

spring-cloud-dataflow-server/src/test/resources/docker-compose-container-registry-it.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+services:
+
+  dataflow-server:
+    environment:
+# AZURE
+      - spring.cloud.dataflow.container.registry-configurations.azure-registry.registry-host=scdftest.azurecr.io
+      - spring.cloud.dataflow.container.registry-configurations.azure-registry.authorization-type=basicauth
+      - spring.cloud.dataflow.container.registry-configurations.azure-registry.user=${CR_AZURE_USER}
+      - spring.cloud.dataflow.container.registry-configurations.azure-registry.secret=${CR_AZURE_PASS}
+# HARBOR
+      - spring.cloud.dataflow.container.registry-configurations.harbor-registry.registry-host=projects.registry.vmware.com
+      - spring.cloud.dataflow.container.registry-configurations.harbor-registry.authorization-type=basicauth
+#      - spring.cloud.dataflow.container.registry-configurations.harbor-registry.authorization-type=dockeroauth2
+      - spring.cloud.dataflow.container.registry-configurations.harbor-registry.user=${CR_HARBOR_USER}
+      - spring.cloud.dataflow.container.registry-configurations.harbor-registry.secret=${CR_HARBOR_PASS}
+# GITHUB
+      - spring.cloud.dataflow.container.registry-configurations.github-registry.registry-host=ghcr.io
+      - spring.cloud.dataflow.container.registry-configurations.github-registry.authorization-type=dockeroauth2
+      - spring.cloud.dataflow.container.registry-configurations.github-registry.user=${CR_GITHUB_USER}
+      - spring.cloud.dataflow.container.registry-configurations.github-registry.secret=${CR_GITHUB_PASS}

src/docker-compose/docker-compose-debug-dataflow.yml

@@ -6,4 +6,4 @@ services:
     ports:
       - "5005:5005"
     environment:
-      - JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
+      - JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005

src/docker-compose/docker-compose-debug-skipper.yml

@@ -6,4 +6,4 @@ services:
     ports:
       - "6006:6006"
     environment:
-      - JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:6006
+      - JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:6006