[JENKINS-75157] Bitbucket client fail to retrieve resource with HTTP 404 for bitbucket server (#970) (#973)

Remove the HttpHost parameter when calling in Apache client as the request could be different than the host configured in jenkins, for example in Bitbucket Server request using mirror link.
Rethrow the FileNotFoundException in executeMethod instead of encapsulate into other exception to respect the SCMFile interface.
Rewrote the BitbucketSCMFile logic to not assume the SCMFile.type but resolve at runtime when needed.

Author: nfalco79

pom.xml

@@ -90,8 +90,6 @@
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>git</artifactId>
-      <!-- TODO: remove when in bom -->
-      <version>5.7.0</version>
     </dependency>
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketApi.java

@@ -302,17 +302,30 @@ List<? extends BitbucketRepository> getRepositories(@CheckForNull UserRoleInRepo
      * @throws IOException if there was a network communications error.
      * @throws InterruptedException if interrupted while waiting on remote communications.
      */
+    @NonNull
     @Restricted(NoExternalUse.class)
     Iterable<SCMFile> getDirectoryContent(BitbucketSCMFile parent) throws IOException, InterruptedException;
 
     /**
      * Return an input stream for the given file.
      *
-     * @param file and instance of SCM file
+     * @param file an instance of SCM file
      * @return the stream of the given {@link SCMFile}
      * @throws IOException if there was a network communications error.
      * @throws InterruptedException if interrupted while waiting on remote communications.
      */
     @Restricted(NoExternalUse.class)
     InputStream getFileContent(BitbucketSCMFile file) throws IOException, InterruptedException;
+
+    /**
+     * Return the metadata for the given file.
+     *
+     * @param file an instance of SCM file
+     * @return a {@link SCMFile} file with updated the metadata
+     * @throws IOException if there was a network communications error.
+     * @throws InterruptedException if interrupted while waiting on remote communications.
+     */
+    @NonNull
+    @Restricted(NoExternalUse.class)
+    SCMFile getFile(@NonNull BitbucketSCMFile file) throws IOException, InterruptedException;
 }

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java

@@ -55,6 +55,7 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import edu.umd.cs.findbugs.annotations.CheckForNull;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import hudson.Util;
 import java.awt.image.BufferedImage;
 import java.io.BufferedInputStream;
@@ -836,33 +837,45 @@ public Iterable<SCMFile> getDirectoryContent(final BitbucketSCMFile parent) thro
                 .set("path", parent.getPath())
                 .expand();
         List<SCMFile> result = new ArrayList<>();
-        String response = getRequest(url);
-        BitbucketCloudPage<BitbucketRepositorySource> page = JsonParser.mapper.readValue(response,
-                new TypeReference<BitbucketCloudPage<BitbucketRepositorySource>>(){});
 
-        for(BitbucketRepositorySource source:page.getValues()){
-            result.add(source.toBitbucketScmFile(parent));
-        }
+        String pageURL = url;
+        BitbucketCloudPage<BitbucketRepositorySource> page;
+        do {
+            String response = getRequest(pageURL);
+            page = JsonParser.mapper.readValue(response, new TypeReference<BitbucketCloudPage<BitbucketRepositorySource>>(){});
 
-        while (!page.isLastPage()){
-            response = getRequest(page.getNext());
-            page = JsonParser.mapper.readValue(response,
-                    new TypeReference<BitbucketCloudPage<BitbucketRepositorySource>>(){});
-            for(BitbucketRepositorySource source:page.getValues()){
-                result.add(source.toBitbucketScmFile(parent));
+            for(BitbucketRepositorySource source : page.getValues()){
+                result.add(source.toBitbucketSCMFile(parent));
             }
-        }
+            pageURL = page.getNext();
+        } while (!page.isLastPage());
         return result;
     }
 
     @Override
-    public InputStream getFileContent(BitbucketSCMFile file) throws IOException, InterruptedException {
-        String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE + "/src{/branchOrHash,path}")
+    public InputStream getFileContent(@NonNull BitbucketSCMFile file) throws IOException, InterruptedException {
+        String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE + "/src{/branchOrHash,path}{?at}")
                 .set("owner", owner)
                 .set("repo", repositoryName)
                 .set("branchOrHash", file.getHash())
                 .set("path", file.getPath())
+                .set("at", file.getRef())
                 .expand();
         return getRequestAsInputStream(url);
     }
+
+    @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
+    @NonNull
+    @Override
+    public SCMFile getFile(@NonNull BitbucketSCMFile file) throws IOException, InterruptedException {
+        String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE + "/src{/branchOrHash,path}?format=meta")
+                .set("owner", owner)
+                .set("repo", repositoryName)
+                .set("branchOrHash", file.getHash() != null ? file.getHash() : file.getRef())
+                .set("path", file.getPath())
+                .expand();
+        String response = getRequest(url);
+        BitbucketRepositorySource src = JsonParser.mapper.readValue(response, BitbucketRepositorySource.class);
+        return src.toBitbucketSCMFile((BitbucketSCMFile) file.parent());
+    }
 }

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/repository/BitbucketRepositorySource.java

@@ -28,6 +28,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.List;
 import java.util.Map;
 import jenkins.scm.api.SCMFile;
@@ -68,23 +69,25 @@ public String getHash() {
 
     @JsonIgnore
     public boolean isDirectory() {
-        return type.equals("commit_directory");
+        return "commit_directory".equals(type);
     }
 
-    public BitbucketSCMFile toBitbucketScmFile(BitbucketSCMFile parent){
+    @NonNull
+    public BitbucketSCMFile toBitbucketSCMFile(BitbucketSCMFile parent) {
         SCMFile.Type fileType;
-        if(isDirectory()){
+        if (isDirectory()) {
             fileType = SCMFile.Type.DIRECTORY;
         } else {
             fileType = SCMFile.Type.REGULAR_FILE;
-            for(String attribute: getAttributes()){
-                if(attribute.equals("link")){
+            for (String attribute : getAttributes()) {
+                if ("link".equals(attribute)) {
                     fileType = SCMFile.Type.LINK;
-                } else if(attribute.equals("subrepository")){
+                } else if ("subrepository".equals(attribute)) {
                     fileType = SCMFile.Type.OTHER; // sub-module or sub-repo
                 }
             }
         }
         return new BitbucketSCMFile(parent, path, fileType, hash);
     }
+
 }

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/filesystem/BitbucketSCMFile.java

@@ -25,16 +25,19 @@
 package com.cloudbees.jenkins.plugins.bitbucket.filesystem;
 
 import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketApi;
+import edu.umd.cs.findbugs.annotations.CheckForNull;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import jenkins.scm.api.SCMFile;
 
-public class BitbucketSCMFile  extends SCMFile {
+public class BitbucketSCMFile extends SCMFile {
 
     private final BitbucketApi api;
     private  String ref;
     private final String hash;
+    private boolean resolved;
 
     public String getRef() {
         return ref;
@@ -44,34 +47,22 @@ public void setRef(String ref) {
         this.ref = ref;
     }
 
-    @Deprecated
-    public BitbucketSCMFile(BitbucketSCMFileSystem bitBucketSCMFileSystem,
-                            BitbucketApi api,
-                            String ref) {
-        this(bitBucketSCMFileSystem, api, ref, null);
-    }
-
-    public BitbucketSCMFile(BitbucketSCMFileSystem bitBucketSCMFileSystem,
-                            BitbucketApi api,
-                            String ref, String hash) {
-        super();
-        type(Type.DIRECTORY);
+    public BitbucketSCMFile(BitbucketApi api, String ref, String hash) {
         this.api = api;
         this.ref = ref;
         this.hash = hash;
+        this.resolved = false;
     }
 
-    @Deprecated
-    public BitbucketSCMFile(@NonNull BitbucketSCMFile parent, String name, Type type) {
-        this(parent, name, type, null);
-    }
-
-    public BitbucketSCMFile(@NonNull BitbucketSCMFile parent, String name, Type type, String hash) {
+    public BitbucketSCMFile(BitbucketSCMFile parent, String name, @CheckForNull Type type, String hash) {
         super(parent, name);
         this.api = parent.api;
         this.ref = parent.ref;
         this.hash = hash;
-        type(type);
+        if (type != null) {
+            type(type);
+        }
+        this.resolved = type != null;
     }
 
     public String getHash() {
@@ -80,40 +71,48 @@ public String getHash() {
 
     @Override
     @NonNull
-    public Iterable<SCMFile> children() throws IOException,
-            InterruptedException {
+    public Iterable<SCMFile> children() throws IOException, InterruptedException {
         if (this.isDirectory()) {
             return api.getDirectoryContent(this);
         } else {
-            throw new IOException("Cannot get children from a regular file");
+            // respect the interface javadoc
+            return Collections.emptyList();
         }
     }
 
     @Override
     @NonNull
     public InputStream content() throws IOException, InterruptedException {
-        if (this.isDirectory()) {
-            throw new IOException("Cannot get raw content from a directory");
-        } else {
+        if (this.isFile()) {
             return api.getFileContent(this);
+        } else {
+            throw new IOException("Cannot get raw content from a directory");
         }
     }
 
     @Override
     public long lastModified() throws IOException, InterruptedException {
-        // TODO: Return valid value when Tag support is implemented
-        return 0;
+        return 0L;
     }
 
     @Override
     @NonNull
     protected SCMFile newChild(String name, boolean assumeIsDirectory) {
-        return new BitbucketSCMFile(this, name, assumeIsDirectory?Type.DIRECTORY:Type.REGULAR_FILE, hash);
+        return new BitbucketSCMFile(this, name, null, hash);
     }
 
     @Override
     @NonNull
     protected Type type() throws IOException, InterruptedException {
+        if (!resolved) {
+            try {
+                SCMFile metadata = api.getFile(this);
+                type(metadata.getType());
+            } catch(IOException e) {
+                type(Type.NONEXISTENT);
+            }
+            resolved = true;
+        }
         return this.getType();
     }
 

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/filesystem/BitbucketSCMFileSystem.java

@@ -43,11 +43,11 @@
 import hudson.Util;
 import hudson.model.Item;
 import hudson.model.Queue;
-import hudson.model.queue.Tasks;
 import hudson.scm.SCM;
 import hudson.scm.SCMDescriptor;
 import hudson.security.ACL;
 import java.io.IOException;
+import java.lang.annotation.Inherited;
 import jenkins.authentication.tokens.api.AuthenticationTokens;
 import jenkins.scm.api.SCMFile;
 import jenkins.scm.api.SCMFileSystem;
@@ -69,21 +69,18 @@ protected BitbucketSCMFileSystem(BitbucketApi api, String ref, SCMRevision rev)
     }
 
     /**
-     * Return timestamp of last commit or of tag if its annotated tag.
-     *
-     * @return timestamp of last commit or of tag if its annotated tag
+     * {@link Inherited}
      */
     @Override
     public long lastModified() throws IOException {
-        // TODO figure out how to implement this
         return 0L;
     }
 
     @NonNull
     @Override
     public SCMFile getRoot() {
         SCMRevision revision = getRevision();
-        return new BitbucketSCMFile(this, api, ref, revision == null ? null : revision.toString());
+        return new BitbucketSCMFile(api, ref, revision == null ? null : revision.toString());
     }
 
     @Extension
@@ -116,22 +113,24 @@ public SCMFileSystem build(@NonNull Item owner, @NonNull SCM scm, @CheckForNull
         }
 
         private static StandardCredentials lookupScanCredentials(@CheckForNull Item context,
-                @CheckForNull String scanCredentialsId, String serverUrl) {
-            if (Util.fixEmpty(scanCredentialsId) == null) {
+                                                                 @CheckForNull String scanCredentialsId,
+                                                                 String serverURL) {
+            scanCredentialsId = Util.fixEmpty(scanCredentialsId);
+            if (scanCredentialsId == null) {
                 return null;
             } else {
                 return CredentialsMatchers.firstOrNull(
-                        CredentialsProvider.lookupCredentials(
+                        CredentialsProvider.lookupCredentialsInItem(
                                 StandardCredentials.class,
                                 context,
-                                context instanceof Queue.Task
-                                        ? Tasks.getDefaultAuthenticationOf((Queue.Task) context)
-                                        : ACL.SYSTEM,
-                                URIRequirementBuilder.fromUri(serverUrl).build()
+                                context instanceof Queue.Task task
+                                        ? task.getDefaultAuthentication2()
+                                        : ACL.SYSTEM2,
+                                URIRequirementBuilder.fromUri(serverURL).build()
                         ),
                         CredentialsMatchers.allOf(
                                 CredentialsMatchers.withId(scanCredentialsId),
-                                AuthenticationTokens.matcher(BitbucketAuthenticator.authenticationContext(serverUrl))
+                                AuthenticationTokens.matcher(BitbucketAuthenticator.authenticationContext(serverURL))
                         )
                 );
             }