Fix binary files failing to upload to GitHub

Author: silvanocerza

package.json

@@ -29,6 +29,7 @@
     "@codemirror/view": "^6.36.2",
     "@uiw/react-codemirror": "^4.23.8",
     "codemirror": "^6.0.1",
+    "file-type": "^20.4.1",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"
   }

pnpm-lock.yaml

@@ -27,6 +27,13 @@ export type NewTreeRequestItem = {
   content?: string;
 };
 
+/**
+ * Response received when we create a new binary blob on GitHub
+ */
+export type CreatedBlob = {
+  sha: string;
+};
+
 /**
  * Represents a git blob response from the GitHub API.
  */
@@ -170,6 +177,36 @@ export default class GithubClient {
     }
   }
 
+  /**
+   * Creates a new blob in the GitHub remote, this is mainly used to upload binary files.
+   *
+   * @param content The content of the blob to upload
+   * @param encoding Content encoding, can be "utf-8" or "base64". Defaults to "base64"
+   * @returns The SHA of the newly uploaded blob
+   */
+  async createBlob(
+    content: string,
+    encoding: "utf-8" | "base64" = "base64",
+  ): Promise<CreatedBlob> {
+    const res = await requestUrl({
+      url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs`,
+      headers: this.headers(),
+      method: "POST",
+      body: JSON.stringify({ content, encoding }),
+      throw: false,
+    });
+    if (res.status < 200 || res.status >= 400) {
+      await this.logger.error("Failed to create blob", res);
+      throw new GithubAPIError(
+        res.status,
+        `Failed to create blob, status ${res.status}`,
+      );
+    }
+    return {
+      sha: res.json["sha"],
+    };
+  }
+
   /**
    * Gets a blob from its sha
    * @param url blob sha

src/github/client.ts

@@ -4,6 +4,7 @@ import {
   normalizePath,
   base64ToArrayBuffer,
   EventRef,
+  arrayBufferToBase64,
 } from "obsidian";
 import GithubClient, {
   GetTreeResponseItem,
@@ -20,6 +21,7 @@ import { GitHubSyncSettings } from "./settings/settings";
 import Logger from "./logger";
 import { decodeBase64String } from "./utils";
 import GitHubSyncPlugin from "./main";
+import { fileTypeFromBuffer } from "file-type";
 
 interface SyncAction {
   type: "upload" | "download" | "delete_local" | "delete_remote";
@@ -666,8 +668,8 @@ export default class SyncManager {
    * @returns String containing the file SHA1
    */
   async calculateSHA(filePath: string): Promise<string> {
-    const content = await this.vault.adapter.read(filePath);
-    const contentBytes = new TextEncoder().encode(content);
+    const contentBuffer = await this.vault.adapter.readBinary(filePath);
+    const contentBytes = new Uint8Array(contentBuffer);
     const header = new TextEncoder().encode(`blob ${contentBytes.length}\0`);
     const store = new Uint8Array([...header, ...contentBytes]);
     return await crypto.subtle.digest("SHA-1", store).then((hash) =>
@@ -713,11 +715,40 @@ export default class SyncManager {
     // We don't save the metadata file after setting the SHAs cause we do that when
     // the sync is fully commited at the end.
     // TODO: Understand whether it's a problem we don't revert the SHA setting in case of sync failure
+    //
+    // In here we also upload blob is file is a binary. We do it here because when uploading a blob we
+    // also get back its SHA, so we can set it together with other files.
+    // We also do that right before creating the new tree because we need the SHAs of those blob to
+    // correctly create it.
     await Promise.all(
       Object.keys(treeFiles)
         .filter((filePath: string) => treeFiles[filePath].content)
         .map(async (filePath: string) => {
-          const newSha = await this.calculateSHA(filePath);
+          const buffer = await this.vault.adapter.readBinary(filePath);
+          const fileType = await fileTypeFromBuffer(buffer);
+          let newSha = "";
+          if (
+            // We can't determine the file type
+            fileType === undefined ||
+            // This is not a text file
+            !fileType.mime.startsWith("text/") ||
+            // Neither a json file
+            fileType.mime !== "application/json"
+          ) {
+            // We treat this file as a binary file. We can't upload these setting the content
+            // of a tree item, we first need to create a Git blob by uploading the file, then
+            // we must update the tree item to point the SHA to the blob we just created.
+            const hash = arrayBufferToBase64(buffer);
+            const { sha } = await this.client.createBlob(hash);
+            treeFiles[filePath].sha = sha;
+            // Can't have both sha and content set, so we delete it
+            delete treeFiles[filePath].content;
+            newSha = sha;
+          } else {
+            // File is text, we can upload the content directly
+            // so we just calculate the new SHA to keep track of it
+            newSha = await this.calculateSHA(filePath);
+          }
           this.metadataStore.data.files[filePath].sha = newSha;
         }),
     );