Enable resolution from private GitHub repositories

atextor · atextor · commit deff172d3e8f · 2024-11-11T11:38:37.000+01:00
Fixes #665
diff --git a/core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/Download.java b/core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/Download.java
@@ -22,7 +22,9 @@
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.time.Duration;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 
@@ -48,9 +50,10 @@ public Download() {
     * Download the file and return the contents as byte array
     *
     * @param fileUrl the URL
+    * @param headers list of additional headers to set
     * @return the file contents
     */
-   public byte[] downloadFile( final URL fileUrl ) {
+   public byte[] downloadFile( final URL fileUrl, final Map<String, String> headers ) {
       try {
          final HttpClient.Builder clientBuilder = HttpClient.newBuilder()
                .version( HttpClient.Version.HTTP_1_1 )
@@ -59,14 +62,31 @@ public byte[] downloadFile( final URL fileUrl ) {
          Optional.ofNullable( proxyConfig.proxy() ).ifPresent( clientBuilder::proxy );
          Optional.ofNullable( proxyConfig.authenticator() ).ifPresent( clientBuilder::authenticator );
          final HttpClient client = clientBuilder.build();
-         final HttpRequest request = HttpRequest.newBuilder().uri( fileUrl.toURI() ).build();
+         final String[] headersArray = headers.entrySet().stream()
+               .flatMap( entry -> Stream.of( entry.getKey(), entry.getValue() ) )
+               .toList()
+               .toArray( new String[0] );
+         final HttpRequest request = HttpRequest.newBuilder()
+               .uri( fileUrl.toURI() )
+               .headers( headersArray )
+               .build();
          final HttpResponse<byte[]> response = client.send( request, HttpResponse.BodyHandlers.ofByteArray() );
          return response.body();
       } catch ( final InterruptedException | URISyntaxException | IOException exception ) {
          throw new ModelResolutionException( "Could not retrieve " + fileUrl, exception );
       }
    }
 
+   /**
+    * Download the file and return the contents as byte array
+    *
+    * @param fileUrl the URL
+    * @return the file contents
+    */
+   public byte[] downloadFile( final URL fileUrl ) {
+      return downloadFile( fileUrl, Map.of() );
+   }
+
    /**
     * Download the file and write it to the file system
     *
@@ -75,8 +95,12 @@ public byte[] downloadFile( final URL fileUrl ) {
     * @return the file written
     */
    public File downloadFile( final URL fileUrl, final File outputFile ) {
+      return downloadFile( fileUrl, Map.of(), outputFile );
+   }
+
+   public File downloadFile( final URL fileUrl, final Map<String, String> headers, final File outputFile ) {
       try ( final FileOutputStream outputStream = new FileOutputStream( outputFile ) ) {
-         final byte[] fileContent = downloadFile( fileUrl );
+         final byte[] fileContent = downloadFile( fileUrl, headers );
          outputStream.write( fileContent );
       } catch ( final IOException exception ) {
          throw new ModelResolutionException( "Could not write file " + outputFile, exception );
diff --git a/core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/GithubRepository.java b/core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/GithubRepository.java
@@ -25,7 +25,7 @@ public record GithubRepository(
       Ref branchOrTag
 ) {
    public GithubRepository( final String owner, final String repository, final Ref branchOrTag ) {
-      this( "github.com", owner, repository, branchOrTag );
+      this( "api.github.com", owner, repository, branchOrTag );
    }
 
    public sealed interface Ref {
@@ -49,8 +49,9 @@ public String refType() {
    }
 
    public URL zipLocation() {
-      final String url = "https://%s/%s/%s/archive/refs/%s/%s.zip".formatted(
-            host(), owner(), repository(), branchOrTag().refType(), branchOrTag().name() );
+      // See https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#download-a-repository-archive-zip
+      // General URL structure: https://api.github.com/repos/OWNER/REPO/zipball/REF
+      final String url = "https://%s/repos/%s/%s/zipball/%s".formatted( host(), owner(), repository(), branchOrTag().name() );
       try {
          return new URL( url );
       } catch ( final MalformedURLException exception ) {
diff --git a/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GitHubModelSource.java b/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GitHubModelSource.java
@@ -21,7 +21,9 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Stream;
 import java.util.zip.ZipFile;
@@ -31,7 +33,7 @@
 import org.eclipse.esmf.aspectmodel.resolver.Download;
 import org.eclipse.esmf.aspectmodel.resolver.GithubRepository;
 import org.eclipse.esmf.aspectmodel.resolver.ModelSource;
-import org.eclipse.esmf.aspectmodel.resolver.ProxyConfig;
+import org.eclipse.esmf.aspectmodel.resolver.modelfile.RawAspectModelFile;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;
 
 import com.google.common.collect.Streams;
@@ -44,51 +46,53 @@
  */
 public class GitHubModelSource implements ModelSource {
    private static final Logger LOG = LoggerFactory.getLogger( GitHubModelSource.class );
-   private final ProxyConfig proxyConfig;
    File repositoryZipFile = null;
    private List<AspectModelFile> files = null;
-   protected final GithubRepository repository;
-   protected final String directory;
+   GithubModelSourceConfig config;
 
-   /**
-    * Constructor.
-    *
-    * @param repository the repository this model sources refers to
-    * @param directory the relative directory inside the repository
-    * @param proxyConfig the proxy configuration
-    */
-   public GitHubModelSource( final GithubRepository repository, final String directory, final ProxyConfig proxyConfig ) {
-      this.repository = repository;
-      this.directory = Optional.ofNullable( directory ).map( d ->
-            d.endsWith( "/" ) ? d.substring( 0, d.length() - 1 ) : d ).orElse( "" );
-      this.proxyConfig = proxyConfig;
+   public GitHubModelSource( final GithubModelSourceConfig config ) {
+      this.config = config;
    }
 
    /**
-    * Constructor. Proxy settings are automatically detected.
+    * Convenience constructor for public repositories. Proxy settings are automatically detected, no authentication is used.
     *
     * @param repository the repository this model sources refers to
     * @param directory the relative directory inside the repository
     */
    public GitHubModelSource( final GithubRepository repository, final String directory ) {
-      this( repository, directory, ProxyConfig.detectProxySettings() );
+      this( GithubModelSourceConfigBuilder.builder()
+            .repository( repository )
+            .directory( directory )
+            .build() );
    }
 
    private String sourceUrl( final String filename ) {
-      return "https://%s/%s/%s/blob/%s/%s".formatted( repository.host(), repository.owner(), repository.repository(),
-            repository.branchOrTag().name(), filename );
+      return "https://%s/%s/%s/blob/%s/%s".formatted(
+            config.repository().host().equals( "api.github.com" ) ? "github.com" : config.repository().host(),
+            config.repository().owner(),
+            config.repository().repository(),
+            config.repository().branchOrTag().name(),
+            filename );
    }
 
    private void init() {
       try {
          final Path tempDirectory = Files.createTempDirectory( "esmf" );
          final File outputZipFile = tempDirectory.resolve( ZonedDateTime.now( ZoneId.systemDefault() )
                .format( DateTimeFormatter.ofPattern( "uuuu-MM-dd.HH.mm.ss" ) ) + ".zip" ).toFile();
-         repositoryZipFile = new Download( proxyConfig ).downloadFile( repository.zipLocation(), outputZipFile );
+         final Map<String, String> headers = new HashMap<>();
+         headers.put( "Accept", "application/vnd.github+json" );
+         headers.put( "X-GitHub-Api-Version", "2022-11-28" );
+         if ( config.token() != null ) {
+            headers.put( "Authorization", "Bearer " + config.token() );
+         }
+         repositoryZipFile = new Download( config.proxyConfig() )
+               .downloadFile( config.repository().zipLocation(), headers, outputZipFile );
          loadFilesFromZip();
          final boolean packageIsDeleted = outputZipFile.delete() && tempDirectory.toFile().delete();
          if ( packageIsDeleted ) {
-            LOG.debug( String.format( "Temporary package file %s was deleted", outputZipFile.getName() ) );
+            LOG.debug( "Temporary package file {} was deleted", outputZipFile.getName() );
          }
       } catch ( final IOException exception ) {
          throw new GitHubResolverException( exception );
@@ -102,21 +106,23 @@ void loadFilesFromZip() {
       try ( final ZipFile zipFile = new ZipFile( repositoryZipFile ) ) {
          LOG.debug( "Loading Aspect Model files from {}", repositoryZipFile );
          files = Streams.stream( zipFile.entries().asIterator() ).flatMap( zipEntry -> {
-            final String pathPrefix = repository.repository() + "-" + repository.branchOrTag().name() + "/" + directory;
-            if ( !zipEntry.getName().startsWith( pathPrefix ) || !zipEntry.getName().endsWith( ".ttl" ) ) {
+            final String path = zipEntry.getName().substring( zipEntry.getName().indexOf( '/' ) + 1 );
+            if ( !zipEntry.getName().endsWith( ".ttl" ) ) {
                return Stream.empty();
             }
-            final int offset = pathPrefix.endsWith( "/" ) ? 0 : 1;
-            final String path = zipEntry.getName().substring( pathPrefix.length() + offset );
-            // Path should now look like org.eclipse.esmf.example/1.0.0/File.ttl
             final String[] parts = path.split( "/" );
-            if ( parts.length != 3 || AspectModelUrn.from( "urn:samm:" + parts[0] + ":" + parts[1] ).isFailure() ) {
+            if ( parts.length < 3 ||
+                  AspectModelUrn.from( "urn:samm:" + parts[parts.length - 3] + ":" + parts[parts.length - 2] ).isFailure() ) {
                LOG.debug( "Tried to load file {} but the path contains no valid URN structure", zipEntry.getName() );
                return Stream.<AspectModelFile> empty();
             }
-            final String relativeFilePath = zipEntry.getName().substring( zipEntry.getName().indexOf( "/" ) + 1 );
-            return Try.of( () -> zipFile.getInputStream( zipEntry ) ).toJavaStream().map( inputStream ->
-                  AspectModelFileLoader.load( inputStream, Optional.of( URI.create( sourceUrl( relativeFilePath ) ) ) ) );
+            final URI uri = URI.create( sourceUrl( path ) );
+            final Try<RawAspectModelFile> file = Try.of( () -> zipFile.getInputStream( zipEntry ) )
+                  .map( inputStream -> AspectModelFileLoader.load( inputStream, Optional.of( uri ) ) );
+            if ( file.isFailure() ) {
+               LOG.debug( "Tried to load {}, but it failed", uri );
+            }
+            return file.toJavaStream();
          } ).toList();
       } catch ( final IOException exception ) {
          throw new GitHubResolverException( exception );
diff --git a/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GitHubStrategy.java b/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GitHubStrategy.java
@@ -15,7 +15,6 @@
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
 import org.eclipse.esmf.aspectmodel.resolver.GithubRepository;
-import org.eclipse.esmf.aspectmodel.resolver.ProxyConfig;
 import org.eclipse.esmf.aspectmodel.resolver.ResolutionStrategy;
 import org.eclipse.esmf.aspectmodel.resolver.ResolutionStrategySupport;
 import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
@@ -33,16 +32,14 @@ public class GitHubStrategy extends GitHubModelSource implements ResolutionStrat
    /**
     * Constructor.
     *
-    * @param repository the GitHub repository
-    * @param directory the relative directory inside the repository
-    * @param proxyConfig the proxy configuration
+    * @param sourceConfig the configuration for the model source
     */
-   public GitHubStrategy( final GithubRepository repository, final String directory, final ProxyConfig proxyConfig ) {
-      super( repository, directory, proxyConfig );
+   public GitHubStrategy( final GithubModelSourceConfig sourceConfig ) {
+      super( sourceConfig );
    }
 
    /**
-    * Constructor. Proxy settings are automatically detected.
+    * Convenience constructor. Proxy settings are automatically detected, no authentication is used.
     *
     * @param repository the GitHub repository
     * @param directory the relative directory inside the repository
@@ -60,8 +57,11 @@ public AspectModelFile apply( final AspectModelUrn aspectModelUrn, final Resolut
             } )
             .filter( file -> resolutionStrategySupport.containsDefinition( file, aspectModelUrn ) )
             .findFirst()
-            .orElseThrow( () -> new ModelResolutionException( "No model file containing " + aspectModelUrn
-                  + " could be found in GitHub repository: " + repository.owner() + "/" + repository.repository()
-                  + " in branch/tag " + repository.branchOrTag().name() ) );
+            .orElseThrow( () -> new ModelResolutionException(
+                  "No model file containing %s could be found in GitHub repository: %s/%s in branch/tag %s".formatted(
+                        aspectModelUrn,
+                        config.repository().owner(),
+                        config.repository().repository(),
+                        config.repository().branchOrTag().name() ) ) );
    }
 }
diff --git a/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GithubModelSourceConfig.java b/core/esmf-aspect-model-github-resolver/src/main/java/org/eclipse/esmf/aspectmodel/resolver/github/GithubModelSourceConfig.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2024 Bosch Software Innovations GmbH. All rights reserved.
+ */
+
+package org.eclipse.esmf.aspectmodel.resolver.github;
+
+import java.util.Optional;
+
+import org.eclipse.esmf.aspectmodel.resolver.GithubRepository;
+import org.eclipse.esmf.aspectmodel.resolver.ProxyConfig;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+
+@RecordBuilder
+public record GithubModelSourceConfig(
+      GithubRepository repository,
+      String directory,
+      ProxyConfig proxyConfig,
+      String token
+) {
+   public GithubModelSourceConfig {
+      directory = Optional.ofNullable( directory ).map( d -> d.endsWith( "/" ) ? d.substring( 0, d.length() - 1 ) : d ).orElse( "" );
+      proxyConfig = Optional.ofNullable( proxyConfig ).orElse( ProxyConfig.detectProxySettings() );
+   }
+}
diff --git a/documentation/developer-guide/modules/tooling-guide/pages/java-aspect-tooling.adoc b/documentation/developer-guide/modules/tooling-guide/pages/java-aspect-tooling.adoc
@@ -181,6 +181,7 @@ model element definitions:
 * The `EitherStrategy` can be used to chain two or more different `ResolutionStrategy`&#8203;s.
 * The `ExternalResolverStrategy` delegates resolution to an external command such as a script; it is used in
   the implementation of the `--custom-resolver` option of the xref:tooling-guide:samm-cli.adoc[samm-cli].
+* The `GitHubStrategy` resolves model elements from repositories hosted on GitHub.
 
 In addition, custom resolution strategies can be provided by implementing the `ResolutionStrategy` interface.
 
diff --git a/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc b/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc
@@ -261,9 +261,11 @@ to load. You can configure where such lookup should be done:
   language/script language, including complex logic if necessary.
 
 * Using the `--github` switch, you can configure a location in a remote GitHub repository as models
-  root, e.g., `eclipse-esmf/esmf-sdk`. Optionally, you can also provide `--github-directory`
-  to set the remote directory and `--github-branch` or `--github-tag` to set the branch name or tag,
-  respectively.
+  root, e.g., `eclipse-esmf/esmf-sdk`. Optionally, you can also provide `--github-directory` to set
+  the remote directory and `--github-branch` or `--github-tag` to set the branch name or tag,
+  respectively. If the repository is private, you can also pass `--github-token` and set a
+  https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token[fine-grained
+  access token]; the token needs at least "Contents" repository read permissions.
 
 NOTE: When using an Aspect Model URN as input to a command (such as
 `urn:samm:org.eclipse.esmf.example:1.0.0#Aspect`), you must also provide at least one of the
diff --git a/tools/samm-cli/src/main/java/org/eclipse/esmf/AbstractInputHandler.java b/tools/samm-cli/src/main/java/org/eclipse/esmf/AbstractInputHandler.java
@@ -27,6 +27,8 @@
 import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.resolver.fs.StructuredModelsRoot;
 import org.eclipse.esmf.aspectmodel.resolver.github.GitHubStrategy;
+import org.eclipse.esmf.aspectmodel.resolver.github.GithubModelSourceConfig;
+import org.eclipse.esmf.aspectmodel.resolver.github.GithubModelSourceConfigBuilder;
 import org.eclipse.esmf.aspectmodel.shacl.violation.Violation;
 import org.eclipse.esmf.aspectmodel.validation.services.AspectModelValidator;
 import org.eclipse.esmf.aspectmodel.validation.services.DetailedViolationFormatter;
@@ -88,7 +90,12 @@ protected List<ResolutionStrategy> configuredStrategies() {
                ? new GithubRepository.Tag( resolverConfig.gitHubResolutionOptions.gitHubTag )
                : new GithubRepository.Branch( resolverConfig.gitHubResolutionOptions.gitHubBranch );
          final GithubRepository repository = new GithubRepository( owner, repositoryName, branchOrTag );
-         strategies.add( new GitHubStrategy( repository, resolverConfig.gitHubResolutionOptions.gitHubDirectory ) );
+         final GithubModelSourceConfig modelSourceConfig = GithubModelSourceConfigBuilder.builder()
+               .repository( repository )
+               .directory( resolverConfig.gitHubResolutionOptions.gitHubDirectory )
+               .token( resolverConfig.gitHubResolutionOptions.gitHubToken )
+               .build();
+         strategies.add( new GitHubStrategy( modelSourceConfig ) );
       }
       return strategies;
    }
diff --git a/tools/samm-cli/src/main/java/org/eclipse/esmf/ResolverConfigurationMixin.java b/tools/samm-cli/src/main/java/org/eclipse/esmf/ResolverConfigurationMixin.java
@@ -44,17 +44,22 @@ public static class GitHubResolutionOptions {
 
       @CommandLine.Option(
             names = { "--github-directory", "-ghd" },
-            description = "Set the GitHub directory (default: ${DEFAULT-VALUE}" )
+            description = "Set the GitHub directory (default: ${DEFAULT-VALUE})" )
       public String gitHubDirectory = "/";
 
       @CommandLine.Option(
             names = { "--github-branch", "-ghb" },
-            description = "Set the GitHub branch (default: ${DEFAULT-VALUE}" )
+            description = "Set the GitHub branch (default: ${DEFAULT-VALUE})" )
       public String gitHubBranch = "main";
 
       @CommandLine.Option(
             names = { "--github-tag", "-ght" },
-            description = "Set the GitHub tag (default: ${DEFAULT-VALUE}" )
+            description = "Set the GitHub tag" )
       public String gitHubTag = null;
+
+      @CommandLine.Option(
+            names = { "--github-token", "-token" },
+            description = "Set the GitHub token" )
+      public String gitHubToken = null;
    }
 }
diff --git a/tools/samm-cli/src/main/java/org/eclipse/esmf/SammCli.java b/tools/samm-cli/src/main/java/org/eclipse/esmf/SammCli.java
@@ -169,6 +169,7 @@ public static void main( final String[] argv ) {
       for ( final String arg : argv ) {
          if ( arg.equals( "--disable-color" ) || arg.equals( "-D" ) ) {
             disableColor = true;
+            break;
          }
       }
 

Original file line number	Diff line number	Diff line change
`@@ -169,6 +169,7 @@ public static void main( final String[] argv ) {`
`169`	`169`	`for ( final String arg : argv ) {`
`170`	`170`	`if ( arg.equals( "--disable-color" ) \|\| arg.equals( "-D" ) ) {`
`171`	`171`	`disableColor = true;`
	`172`	`+ break;`
`172`	`173`	`}`
`173`	`174`	`}`
`174`	`175`