Add support for private/gated model access (Closes #198) (#202)

* Allow user to specify HF token as an environment variable

* Add documentation for how to make authorized requests

* Improve docs

Author: xenova

docs/source/_toctree.yml

@@ -20,6 +20,10 @@
   - local: tutorials/node-audio-processing
     title: Server-side Audio Processing in Node.js
   title: Tutorials
+- sections:
+  - local: guides/private
+    title: Accessing Private/Gated Models
+  title: Developer Guides
 - sections:
   - local: api/transformers
     title: Index

docs/source/guides/private.mdx

@@ -0,0 +1,63 @@
+
+# Accessing Private/Gated Models
+
+<Tip>
+
+Due to the possibility of leaking access tokens to users of your website or web application, we only support accessing private/gated models from server-side environments (e.g., Node.js) that have access to the process' environment variables.
+
+</Tip>
+
+## Step 1: Generating a User Access Token
+
+[User Access Tokens](https://huggingface.co/docs/hub/security-tokens) are the preferred way to authenticate an application to Hugging Face services.
+
+To generate an access token, navigate to the [Access Tokens tab](https://huggingface.co/settings/tokens) in your settings and click on the **New token** button. Choose a name for your token and click **Generate a token** (we recommend keeping the "Role" as read-only). You can then click the **Copy** button next to your newly-created token to copy it to your clipboard. 
+
+<div class="flex justify-center">
+<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/new-token.png"/>
+<img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/new-token-dark.png"/>
+</div>
+
+To delete or refresh User Access Tokens, you can click the **Manage** button.
+
+
+## Step 2: Using the access token in Transformers.js
+
+Transformers.js will attach an Authorization header to requests made to the Hugging Face Hub when the `HF_ACCESS_TOKEN` environment variable is set and visible to the process.
+
+One way to do this is to call your program with the environment variable set. For example, let's say you have a file called `llama.js` with the following code:
+
+```js
+import { AutoTokenizer } from '@xenova/transformers';
+
+// Load tokenizer for a gated repository.
+const tokenizer = await AutoTokenizer.from_pretrained('meta-llama/Llama-2-7b-hf');
+
+// Encode text.
+const text = 'Hello world!';
+const encoded = tokenizer.encode(text);
+console.log(encoded);
+```
+
+You can then use the following command to set the `HF_ACCESS_TOKEN` environment variable and run the file:
+
+```bash
+HF_ACCESS_TOKEN=hf_... node tests/llama.js
+```
+
+(remember to replace `hf_...` with your actual access token).
+
+If done correctly, you should see the following output:
+
+```bash
+[ 1, 15043, 3186, 29991 ]
+```
+
+
+Alternatively, you can set the environment variable directly in your code:
+```js
+// Set access token (NB: Keep this private!)
+process.env.HF_ACCESS_TOKEN = 'hf_...';
+
+// ... rest of your code
+```

docs/source/index.mdx

@@ -17,10 +17,11 @@
 
 ## Contents
 
-The documentation is organized into 3 sections:
+The documentation is organized into 4 sections:
 1. **GET STARTED** provides a quick tour of the library and installation instructions to get up and running.
 2. **TUTORIALS** are a great place to start if you're a beginner! We also include sample applications for you to play around with!
-3. **API REFERENCE** describes all classes and functions, as well as their available parameters and types.
+3. **HOW-TO GUIDES** show you how to use the library to achieve a specific goal.
+4. **API REFERENCE** describes all classes and functions, as well as their available parameters and types.
 
 ## Supported tasks/models
 

src/utils/hub.js

@@ -31,21 +31,6 @@ if (!globalThis.ReadableStream) {
  * NOTE: This setting is ignored for local requests.
  */
 
-class Headers extends Object {
-    constructor(...args) {
-        super();
-        Object.assign(this, args);
-    }
-
-    get(key) {
-        return this[key];
-    }
-
-    clone() {
-        return new Headers(this);
-    }
-}
-
 class FileResponse {
     /**
      * Mapping from file extensions to MIME types.
@@ -75,7 +60,7 @@ class FileResponse {
             this.statusText = 'OK';
 
             let stats = fs.statSync(filePath);
-            this.headers['content-length'] = stats.size;
+            this.headers.set('content-length', stats.size.toString());
 
             this.updateContentType();
 
@@ -103,7 +88,7 @@ class FileResponse {
     updateContentType() {
         // Set content-type header based on file extension
         const extension = this.filePath.toString().split('.').pop().toLowerCase();
-        this.headers['content-type'] = this._CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream';
+        this.headers.set('content-type', this._CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream');
     }
 
     /**
@@ -115,7 +100,7 @@ class FileResponse {
         response.exists = this.exists;
         response.status = this.status;
         response.statusText = this.statusText;
-        response.headers = this.headers.clone();
+        response.headers = new Headers(this.headers);
         return response;
     }
 
@@ -138,7 +123,7 @@ class FileResponse {
      */
     async blob() {
         const data = await fs.promises.readFile(this.filePath);
-        return new Blob([data], { type: this.headers['content-type'] });
+        return new Blob([data], { type: this.headers.get('content-type') });
     }
 
     /**
@@ -167,16 +152,20 @@ class FileResponse {
 /**
  * Determines whether the given string is a valid HTTP or HTTPS URL.
  * @param {string|URL} string The string to test for validity as an HTTP or HTTPS URL.
+ * @param {string[]} [validHosts=null] A list of valid hostnames. If specified, the URL's hostname must be in this list.
  * @returns {boolean} True if the string is a valid HTTP or HTTPS URL, false otherwise.
  */
-function isValidHttpUrl(string) {
+function isValidHttpUrl(string, validHosts = null) {
     // https://stackoverflow.com/a/43467144
     let url;
     try {
         url = new URL(string);
     } catch (_) {
         return false;
     }
+    if (validHosts && !validHosts.includes(url.hostname)) {
+        return false;
+    }
     return url.protocol === "http:" || url.protocol === "https:";
 }
 
@@ -194,13 +183,25 @@ export async function getFile(urlOrPath) {
     } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {
         const IS_CI = !!process.env?.TESTING_REMOTELY;
         const version = env.version;
-        return fetch(urlOrPath, {
-            headers: {
-                'User-Agent': `transformers.js/${version}; is_ci/${IS_CI};`
+
+        const headers = new Headers();
+        headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`);
+
+        // Check whether we are making a request to the Hugging Face Hub.
+        const isHFURL = isValidHttpUrl(urlOrPath, ['huggingface.co', 'hf.co']);
+        if (isHFURL) {
+            // If an access token is present in the environment variables,
+            // we add it to the request headers.
+            const token = process.env?.HF_ACCESS_TOKEN;
+            if (token) {
+                headers.set('Authorization', `Bearer ${token}`);
             }
-        });
+        }
+        return fetch(urlOrPath, { headers });
     } else {
         // Running in a browser-environment, so we use default headers
+        // NOTE: We do not allow passing authorization headers in the browser,
+        // since this would require exposing the token to the client.
         return fetch(urlOrPath);
     }
 }
@@ -409,11 +410,10 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti
     if (response === undefined) {
         // Caching not available, or file is not cached, so we perform the request
 
-        let isURL = isValidHttpUrl(requestURL);
-
         if (env.allowLocalModels) {
             // Accessing local models is enabled, so we try to get the file locally.
             // If request is a valid HTTP URL, we skip the local file check. Otherwise, we try to get the file locally.
+            const isURL = isValidHttpUrl(requestURL);
             if (!isURL) {
                 try {
                     response = await getFile(localPath);