Skip to content

Commit 7cfef3c

Browse files
atextoratextor
committed
Refactor downloading and GH URL parsing in GitHub model source
1 parent 53f16ac commit 7cfef3c

File tree

12 files changed

+489
-121
lines changed

12 files changed

+489
-121
lines changed

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/AspectModelFileLoader.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import static org.apache.commons.lang3.StringUtils.isBlank;
1717

1818
import java.io.BufferedReader;
19+
import java.io.ByteArrayInputStream;
1920
import java.io.File;
2021
import java.io.FileInputStream;
2122
import java.io.FileNotFoundException;
@@ -97,15 +98,24 @@ public static RawAspectModelFile load( final Model model ) {
9798
return new RawAspectModelFile( model, List.of(), Optional.empty() );
9899
}
99100

101+
public static RawAspectModelFile load( final byte[] content ) {
102+
return load( new ByteArrayInputStream( content ) );
103+
}
104+
100105
public static RawAspectModelFile load( final URL url ) {
101106
if ( url.getProtocol().equals( "file" ) ) {
102107
try {
103108
return load( Paths.get( url.toURI() ).toFile() );
104109
} catch ( final URISyntaxException exception ) {
105110
throw new ModelResolutionException( "Can not load model from file URL", exception );
106111
}
112+
} else if ( url.getProtocol().equals( "http" ) || url.getProtocol().equals( "https" ) ) {
113+
// Downloading from http(s) should take proxy settings into consideration, so we don't just .openStream() here
114+
final byte[] fileContent = new Download().downloadFile( url );
115+
return load( fileContent );
107116
}
108117
try {
118+
// Other URLs (e.g. resource://) we just load using openStream()
109119
return load( url.openStream(), Optional.of( url.toURI() ) );
110120
} catch ( final IOException | URISyntaxException exception ) {
111121
throw new ModelResolutionException( "Can not load model from URL", exception );

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/Download.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
3+
*
4+
* See the AUTHORS file(s) distributed with this work for additional
5+
* information regarding authorship.
6+
*
7+
* This Source Code Form is subject to the terms of the Mozilla Public
8+
* License, v. 2.0. If a copy of the MPL was not distributed with this
9+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
10+
*
11+
* SPDX-License-Identifier: MPL-2.0
12+
*/
13+
14+
package org.eclipse.esmf.aspectmodel.resolver;
15+
16+
import java.io.File;
17+
import java.io.FileOutputStream;
18+
import java.io.IOException;
19+
import java.net.URISyntaxException;
20+
import java.net.URL;
21+
import java.net.http.HttpClient;
22+
import java.net.http.HttpRequest;
23+
import java.net.http.HttpResponse;
24+
import java.time.Duration;
25+
import java.util.Optional;
26+
27+
import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
28+
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
/**
33+
* Convenience class to download a file via HTTP, which the ability to auto-detect and use proxy settings
34+
*/
35+
public class Download {
36+
private static final Logger LOG = LoggerFactory.getLogger( Download.class );
37+
private final ProxyConfig proxyConfig;
38+
39+
public Download( final ProxyConfig proxyConfig ) {
40+
this.proxyConfig = proxyConfig;
41+
}
42+
43+
public Download() {
44+
this( ProxyConfig.detectProxySettings() );
45+
}
46+
47+
/**
48+
* Download the file and return the contents as byte array
49+
*
50+
* @param fileUrl the URL
51+
* @return the file contents
52+
*/
53+
public byte[] downloadFile( final URL fileUrl ) {
54+
try {
55+
final HttpClient.Builder clientBuilder = HttpClient.newBuilder()
56+
.version( HttpClient.Version.HTTP_1_1 )
57+
.followRedirects( HttpClient.Redirect.ALWAYS )
58+
.connectTimeout( Duration.ofSeconds( 10 ) );
59+
Optional.ofNullable( proxyConfig.proxy() ).ifPresent( clientBuilder::proxy );
60+
Optional.ofNullable( proxyConfig.authenticator() ).ifPresent( clientBuilder::authenticator );
61+
final HttpClient client = clientBuilder.build();
62+
final HttpRequest request = HttpRequest.newBuilder().uri( fileUrl.toURI() ).build();
63+
final HttpResponse<byte[]> response = client.send( request, HttpResponse.BodyHandlers.ofByteArray() );
64+
return response.body();
65+
} catch ( final InterruptedException | URISyntaxException | IOException exception ) {
66+
throw new ModelResolutionException( "Could not retrieve " + fileUrl, exception );
67+
}
68+
}
69+
70+
/**
71+
* Download the file and write it to the file system
72+
*
73+
* @param fileUrl the URL
74+
* @param outputFile the output file to write
75+
* @return the file written
76+
*/
77+
public File downloadFile( final URL fileUrl, final File outputFile ) {
78+
try ( final FileOutputStream outputStream = new FileOutputStream( outputFile ) ) {
79+
final byte[] fileContent = downloadFile( fileUrl );
80+
outputStream.write( fileContent );
81+
} catch ( final IOException exception ) {
82+
throw new ModelResolutionException( "Could not write file " + outputFile, exception );
83+
}
84+
85+
LOG.info( "Downloaded {} to local file {}", fileUrl.getPath(), outputFile );
86+
return outputFile;
87+
}
88+
}

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/GitHubFileLocation.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
3+
*
4+
* See the AUTHORS file(s) distributed with this work for additional
5+
* information regarding authorship.
6+
*
7+
* This Source Code Form is subject to the terms of the Mozilla Public
8+
* License, v. 2.0. If a copy of the MPL was not distributed with this
9+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
10+
*
11+
* SPDX-License-Identifier: MPL-2.0
12+
*/
13+
14+
package org.eclipse.esmf.aspectmodel.resolver;
15+
16+
import java.io.Serial;
17+
import java.util.Arrays;
18+
import java.util.Optional;
19+
import java.util.function.Supplier;
20+
21+
import io.vavr.control.Try;
22+
23+
public record GitHubFileLocation(
24+
GithubRepository repositoryLocation,
25+
String directory,
26+
String namespaceMainPart,
27+
String version,
28+
String filename
29+
) {
30+
public static Optional<GitHubFileLocation> parse( final String url ) {
31+
return Try.of( () -> new GitHubUrlParser( url ).get() ).toJavaOptional();
32+
}
33+
34+
private static class ParsingException extends RuntimeException {
35+
@Serial
36+
private static final long serialVersionUID = -4855068216382713797L;
37+
}
38+
39+
private static class GitHubUrlParser implements Supplier<GitHubFileLocation> {
40+
private int index = 0;
41+
private final String source;
42+
43+
private GitHubUrlParser( final String source ) {
44+
this.source = source;
45+
}
46+
47+
private String readSection() {
48+
final int oldIndex = index;
49+
while ( index < source.length() && source.charAt( index ) != '/' ) {
50+
index++;
51+
}
52+
if ( source.charAt( index ) != '/' ) {
53+
throw new ParsingException();
54+
}
55+
final String result = source.substring( oldIndex, index );
56+
index++; // eat the slash
57+
return result;
58+
}
59+
60+
private void eatToken( final String token ) {
61+
if ( source.startsWith( token, index ) ) {
62+
index += token.length();
63+
return;
64+
}
65+
throw new ParsingException();
66+
}
67+
68+
@Override
69+
public GitHubFileLocation get() {
70+
eatToken( "https://" );
71+
final String host = readSection();
72+
final String owner = readSection();
73+
final String repository = readSection();
74+
final GithubRepository.Ref ref;
75+
if ( source.substring( index ).startsWith( "blob" ) ) {
76+
eatToken( "blob/" );
77+
final String blob = readSection();
78+
ref = blob.matches( "[vV]?\\d+\\.\\d+.*" )
79+
? new GithubRepository.Tag( blob )
80+
: new GithubRepository.Branch( blob );
81+
} else {
82+
eatToken( "raw/refs/" );
83+
if ( source.substring( index ).startsWith( "heads" ) ) {
84+
eatToken( "heads/" );
85+
final String branchName = readSection();
86+
ref = new GithubRepository.Branch( branchName );
87+
} else {
88+
eatToken( "tags/" );
89+
final String tag = readSection();
90+
ref = new GithubRepository.Tag( tag );
91+
}
92+
}
93+
final String rest = source.substring( index );
94+
final String[] parts = rest.split( "/" );
95+
final String fileName = parts[parts.length - 1];
96+
final String version = parts[parts.length - 2];
97+
final String namespaceMainPart = parts[parts.length - 3];
98+
final String directory = String.join( "/", Arrays.copyOfRange( parts, 0, parts.length - 3 ) );
99+
final GithubRepository repoLocation = new GithubRepository( host, owner, repository, ref );
100+
return new GitHubFileLocation( repoLocation, directory, namespaceMainPart, version, fileName );
101+
}
102+
}
103+
}

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/GithubRepository.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
3+
*
4+
* See the AUTHORS file(s) distributed with this work for additional
5+
* information regarding authorship.
6+
*
7+
* This Source Code Form is subject to the terms of the Mozilla Public
8+
* License, v. 2.0. If a copy of the MPL was not distributed with this
9+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
10+
*
11+
* SPDX-License-Identifier: MPL-2.0
12+
*/
13+
14+
package org.eclipse.esmf.aspectmodel.resolver;
15+
16+
import java.net.MalformedURLException;
17+
import java.net.URL;
18+
19+
import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
20+
21+
public record GithubRepository(
22+
String host,
23+
String owner,
24+
String repository,
25+
Ref branchOrTag
26+
) {
27+
public GithubRepository( final String owner, final String repository, final Ref branchOrTag ) {
28+
this( "github.com", owner, repository, branchOrTag );
29+
}
30+
31+
public sealed interface Ref {
32+
String name();
33+
34+
String refType();
35+
}
36+
37+
public record Branch( String name ) implements Ref {
38+
@Override
39+
public String refType() {
40+
return "heads";
41+
}
42+
}
43+
44+
public record Tag( String name ) implements Ref {
45+
@Override
46+
public String refType() {
47+
return "tags";
48+
}
49+
}
50+
51+
public URL zipLocation() {
52+
final String url = "https://%s/%s/%s/archive/refs/%s/%s.zip".formatted(
53+
host(), owner(), repository(), branchOrTag().refType(), branchOrTag().name() );
54+
try {
55+
return new URL( url );
56+
} catch ( final MalformedURLException exception ) {
57+
throw new ModelResolutionException( "Constructed GitHub URL is invalid: " + url );
58+
}
59+
}
60+
}

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/ProxyConfig.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
3+
*
4+
* See the AUTHORS file(s) distributed with this work for additional
5+
* information regarding authorship.
6+
*
7+
* This Source Code Form is subject to the terms of the Mozilla Public
8+
* License, v. 2.0. If a copy of the MPL was not distributed with this
9+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
10+
*
11+
* SPDX-License-Identifier: MPL-2.0
12+
*/
13+
14+
package org.eclipse.esmf.aspectmodel.resolver;
15+
16+
import java.net.Authenticator;
17+
import java.net.InetSocketAddress;
18+
import java.net.ProxySelector;
19+
import java.util.regex.Matcher;
20+
import java.util.regex.Pattern;
21+
22+
import io.soabase.recordbuilder.core.RecordBuilder;
23+
import org.slf4j.Logger;
24+
import org.slf4j.LoggerFactory;
25+
26+
@RecordBuilder
27+
public record ProxyConfig(
28+
ProxySelector proxy,
29+
Authenticator authenticator
30+
) {
31+
private static final Logger LOG = LoggerFactory.getLogger( ProxyConfig.class );
32+
33+
public static final ProxyConfig NO_PROXY = new ProxyConfig( null, null );
34+
35+
public static ProxyConfig detectProxySettings() {
36+
final String envProxy = System.getenv( "http_proxy" );
37+
if ( envProxy != null && System.getProperty( "http.proxyHost" ) == null ) {
38+
final Pattern proxyPattern = Pattern.compile( "http://([^:]*):(\\d+)/?" );
39+
final Matcher matcher = proxyPattern.matcher( envProxy );
40+
if ( matcher.matches() ) {
41+
final String host = matcher.group( 1 );
42+
final String port = matcher.group( 2 );
43+
System.setProperty( "http.proxyHost", host );
44+
System.setProperty( "http.proxyPort", port );
45+
} else {
46+
LOG.debug( "The value of the 'http_proxy' environment variable is malformed, ignoring: {}", envProxy );
47+
}
48+
}
49+
50+
final String host = System.getProperty( "http.proxyHost" );
51+
final String port = System.getProperty( "http.proxyPort" );
52+
if ( host != null && port != null ) {
53+
return new ProxyConfig( ProxySelector.of( new InetSocketAddress( host, Integer.parseInt( port ) ) ), null );
54+
}
55+
return NO_PROXY;
56+
}
57+
}

‎core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/fs/StructuredModelsRoot.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public Stream<URI> contents() {
6767
.flatMap( versionNumber -> AspectModelUrn.from( String.format( "urn:samm:%s:%s",
6868
file.getParentFile().getParentFile().getName(), versionNumber ) ) ).isSuccess() )
6969
.sorted( Comparator.comparing( File::getName ) )
70+
.map( File::getAbsoluteFile )
7071
.map( File::toURI )
7172
.toList()
7273
.stream();
@@ -78,8 +79,12 @@ public Stream<URI> contents() {
7879

7980
@Override
8081
public Stream<URI> namespaceContents( final AspectModelUrn namespace ) {
81-
final File namespaceDirectory = rootPath().resolve( namespace.getNamespaceMainPart() ).resolve( namespace.getVersion() ).toFile();
82-
return Arrays.stream( Objects.requireNonNull( namespaceDirectory.listFiles( file -> file.getName().endsWith( ".ttl" ) ) ) )
82+
final File namespaceDirectory = rootPath()
83+
.resolve( namespace.getNamespaceMainPart() )
84+
.resolve( namespace.getVersion() )
85+
.toFile();
86+
return Arrays.stream( Objects.requireNonNull( namespaceDirectory.listFiles( file ->
87+
file.getName().endsWith( ".ttl" ) ) ) )
8388
.map( File::toURI );
8489
}
8590
}

‎core/esmf-aspect-meta-model-java/src/test/java/org/eclipse/esmf/aspectmodel/resolver/AspectModelResolverTest.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,7 @@
1919
import static org.assertj.core.api.Assertions.assertThatThrownBy;
2020

2121
import java.io.File;
22-
import java.io.IOException;
23-
import java.io.InputStream;
2422
import java.net.URISyntaxException;
25-
import java.net.URL;
26-
import java.nio.charset.StandardCharsets;
27-
import java.util.Arrays;
2823

2924
import org.eclipse.esmf.aspectmodel.loader.AspectModelLoader;
3025
import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
@@ -35,7 +30,6 @@
3530
import org.eclipse.esmf.samm.KnownVersion;
3631
import org.eclipse.esmf.test.TestModel;
3732

38-
import org.apache.commons.io.IOUtils;
3933
import org.apache.jena.rdf.model.Model;
4034
import org.apache.jena.rdf.model.Resource;
4135
import org.apache.jena.vocabulary.RDF;

0 commit comments

Comments
 (0)