Clients: add modify and clear (#23)

* add modify and clear

* test blyss service via python client

Author: neilmovva

.github/workflows/build-python.yml

@@ -1,5 +1,9 @@
 name: Build Python SDK
 
+env:
+  BLYSS_STAGING_SERVER: https://dev2.api.blyss.dev
+  BLYSS_STAGING_API_KEY: Gh1pz1kEiNa1npEdDaRRvM1LsVypM1u2x1YbGb54
+
 on:
   push:
     branches: [ "main" ]
@@ -26,6 +30,14 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
+      - name: Install Python SDK
+        working-directory: python
+        shell: bash
+        run: pip install .
+      - name: Test Python SDK
+        working-directory: python
+        shell: bash
+        run: python tests/test_service.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "python.analysis.typeCheckingMode": "basic"
+    "python.analysis.typeCheckingMode": "basic",
+    "editor.formatOnSave": true
 }

e2e-tests/api.ts

@@ -0,0 +1,129 @@
+import type { Bucket, Client } from '@blyss/sdk';
+const blyss = require('@blyss/sdk/node');
+
+async function keyToValue(key: string, len: number): Promise<Uint8Array> {
+    const keyBytes = new TextEncoder().encode(key);
+    const value = new Uint8Array(len);
+    let i = 0;
+    // fill the value with the hash.
+    // if the hash is smaller than the value, we hash the hash again.
+    while (i < len) {
+        const hash = await crypto.subtle.digest('SHA-1', keyBytes);
+        const hashBytes = new Uint8Array(hash);
+        const toCopy = Math.min(hashBytes.length, len - i);
+        value.set(hashBytes.slice(0, toCopy), i);
+        i += toCopy;
+    }
+    return value;
+}
+
+async function verifyRead(key: string, value: Uint8Array): Promise<void> {
+    const expected = await keyToValue(key, value.length);
+    if (expected.toString() !== value.toString()) {
+        throw new Error('Incorrect value for key ' + key);
+    }
+}
+
+function generateKeys(n: number, seed: number = 0): string[] {
+    return new Array(n).fill(0).map(
+        (_, i) => seed.toString() + '-' + i.toString()
+    );
+}
+
+function generateBucketName(): string {
+    return 'api-tester-' + Math.random().toString(16).substring(2, 10);
+}
+
+async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev') {
+    const apiKey = process.env.BLYSS_API_KEY;
+    if (!apiKey) {
+        throw new Error('BLYSS_API_KEY environment variable is not set');
+    }
+    console.log('Using key: ' + apiKey + ' to connect to ' + endpoint);
+    const client: Client = await new blyss.Client(
+        {
+            endpoint: endpoint,
+            apiKey: apiKey
+        }
+    );
+    // generate random string for bucket name
+    const bucketName = generateBucketName();
+    await client.create(bucketName);
+    const bucket: Bucket = await client.connect(bucketName);
+    console.log(bucket.metadata);
+
+    // generate N random keys
+    const N = 100;
+    const itemSize = 32;
+    let localKeys = generateKeys(N);
+    function getRandomKey(): string {
+        return localKeys[Math.floor(Math.random() * localKeys.length)];
+    }
+    // write all N keys
+    await bucket.write(
+        await Promise.all(localKeys.map(
+            async (k) => ({
+                k: await keyToValue(k, itemSize)
+            })
+        ))
+    );
+    console.log(`Wrote ${N} keys`);
+
+    // read a random key
+    let testKey = getRandomKey();
+    let value = await bucket.privateRead(testKey);
+    await verifyRead(testKey, value);
+    console.log(`Read key ${testKey}`);
+
+    // delete testKey from the bucket, and localData.
+    await bucket.deleteKey(testKey);
+    localKeys.splice(localKeys.indexOf(testKey), 1);
+    console.log(`Deleted key ${testKey}`);
+
+    // write a new value
+    testKey = 'newKey0';
+    await bucket.write({ testKey: keyToValue(testKey, itemSize) });
+    localKeys.push(testKey);
+    console.log(`Wrote key ${testKey}`);
+
+    // clear all keys
+    await bucket.clearEntireBucket();
+    localKeys = [];
+    console.log('Cleared bucket');
+
+    // write a new set of N keys
+    localKeys = generateKeys(N, 1);
+    await bucket.write(
+        await Promise.all(localKeys.map(
+            async (k) => ({
+                k: await keyToValue(k, itemSize)
+            })
+        ))
+    );
+    console.log(`Wrote ${N} keys`);
+
+    // rename the bucket
+    const newBucketName = bucketName + '-rn';
+    await bucket.rename(newBucketName);
+    console.log(`Renamed bucket`);
+    console.log(await bucket.info());
+
+    // random read
+    testKey = getRandomKey();
+    value = await bucket.privateRead(testKey);
+    await verifyRead(testKey, value);
+    console.log(`Read key ${testKey}`);
+
+    // destroy the bucket
+    await bucket.destroyEntireBucket();
+    console.log(`Destroyed bucket ${bucket.name}`);
+}
+
+async function main() {
+    const endpoint = "https://dev2.api.blyss.dev"
+    console.log('Testing Blyss service at URL ' + endpoint);
+    await testBlyssService(endpoint);
+    console.log('All tests completed successfully.');
+}
+
+main();

js/bucket/bucket.ts

@@ -3,7 +3,6 @@ import { base64ToBytes, getRandomSeed } from '../client/seed';
 import { decompress } from '../compression/bz2_decompress';
 import { bloomLookup } from '../data/bloom';
 import {
-  DataWithMetadata,
   concatBytes,
   deserialize,
   deserializeChunks,
@@ -40,7 +39,7 @@ export class Bucket {
   readonly api: Api;
 
   /** The name of this bucket. */
-  readonly name: string;
+  name: string;
 
   /**
    * The secret seed for this instance of the client, which can be saved and
@@ -110,7 +109,7 @@ export class Bucket {
     try {
       decompressedResult = decompress(decryptedResult);
     } catch (e) {
-      console.error('decompress error', e);
+      console.error(`key ${key} not found (decompression failed)`);
     }
     if (decompressedResult === null) {
       return null;
@@ -120,7 +119,7 @@ export class Bucket {
     try {
       extractedResult = this.lib.extractResult(key, decompressedResult);
     } catch (e) {
-      console.error('extraction error', e);
+      console.error(`key ${key} not found (extraction failed)`);
     }
     if (extractedResult === null) {
       return null;
@@ -151,7 +150,7 @@ export class Bucket {
 
   private async performPrivateReads(
     keys: string[]
-  ): Promise<DataWithMetadata[]> {
+  ): Promise<any[]> {
     if (!this.uuid || !this.check(this.uuid)) {
       await this.setup();
     }
@@ -185,7 +184,7 @@ export class Bucket {
     return endResults;
   }
 
-  private async performPrivateRead(key: string): Promise<DataWithMetadata> {
+  private async performPrivateRead(key: string): Promise<any> {
     return (await this.performPrivateReads([key]))[0];
   }
 
@@ -320,6 +319,15 @@ export class Bucket {
     return await this.api.meta(this.name);
   }
 
+  /** Renames this bucket, leaving all data and other bucket settings intact. */
+  async rename(newBucketName: string): Promise<BucketMetadata> {
+    const bucketCreateReq = {
+      name: newBucketName
+    };
+    await this.api.modify(this.name, JSON.stringify(bucketCreateReq));
+    this.name = newBucketName;
+  }
+
   /** Gets info on all keys in this bucket. */
   async listKeys(): Promise<KeyInfo[]> {
     this.ensureSpiral();
@@ -333,32 +341,28 @@ export class Bucket {
    *   key-value pairs to write. Keys must be strings, and values may be any
    *   JSON-serializable value or a Uint8Array. The maximum size of a key is
    *   1024 UTF-8 bytes.
-   * @param {{ [key: string]: any }} [metadata] - An optional object containing
-   *   metadata. Each key of this object should also be a key of
-   *   `keyValuePairs`, and the value should be some metadata object to store
-   *   with the values being written.
    */
   async write(
-    keyValuePairs: { [key: string]: any },
-    metadata?: { [key: string]: any }
+    keyValuePairs: { [key: string]: any }
   ) {
     this.ensureSpiral();
 
     const data = [];
     for (const key in keyValuePairs) {
       if (Object.prototype.hasOwnProperty.call(keyValuePairs, key)) {
         const value = keyValuePairs[key];
-        let valueMetadata = undefined;
-        if (metadata && Object.prototype.hasOwnProperty.call(metadata, key)) {
-          valueMetadata = metadata[key];
-        }
-        const valueBytes = serialize(value, valueMetadata);
+        const valueBytes = serialize(value);
         const keyBytes = new TextEncoder().encode(key);
         const serializedKeyValue = wrapKeyValue(keyBytes, valueBytes);
         data.push(serializedKeyValue);
+        // const kv = {
+        //   key: key,
+        //   value: Buffer.from(valueBytes).toString('base64')
+        // }
       }
     }
     const concatenatedData = concatBytes(data);
+    // const concatenatedData = serialize(data);
     await this.api.write(this.name, concatenatedData);
   }
 
@@ -385,6 +389,14 @@ export class Bucket {
     await this.api.destroy(this.name);
   }
 
+  /**
+   * Clears the contents of the entire bucket, and all data inside of it. This action is
+   * permanent and irreversible.
+   */
+  async clearEntireBucket() {
+    await this.api.clear(this.name);
+  }
+
   /**
    * Privately reads the supplied key from the bucket, returning the value
    * corresponding to the key.
@@ -398,28 +410,13 @@ export class Bucket {
     this.ensureSpiral();
 
     if (Array.isArray(key)) {
-      return (await this.performPrivateReads(key)).map(r => r.data);
+      return (await this.performPrivateReads(key));
     } else {
       const result = await this.performPrivateRead(key);
-      return result ? result.data : null;
+      return result ? result : null;
     }
   }
 
-  /**
-   * Privately reads the supplied key from the bucket, returning the value and
-   * metadata corresponding to the key.
-   *
-   * No entity, including the Blyss service, should be able to determine which
-   * key this method was called for.
-   *
-   * @param {string} key - The key to _privately_ retrieve the value of.
-   */
-  async privateReadWithMetadata(key: string): Promise<DataWithMetadata> {
-    this.ensureSpiral();
-
-    return await this.performPrivateRead(key);
-  }
-
   /**
    * Privately intersects the given set of keys with the keys in this bucket,
    * returning the keys that intersected and their values. This is generally
@@ -437,7 +434,7 @@ export class Bucket {
     this.ensureSpiral();
 
     if (keys.length < BLOOM_CUTOFF) {
-      return (await this.performPrivateReads(keys)).map(x => x.data);
+      return (await this.performPrivateReads(keys));
     }
 
     const bloomFilter = await this.api.bloom(this.name);
@@ -451,7 +448,7 @@ export class Bucket {
     if (!retrieveValues) {
       return matches;
     }
-    return (await this.performPrivateReads(matches)).map(x => x.data);
+    return (await this.performPrivateReads(matches));
   }
 
   /**

js/client/api.ts

@@ -4,6 +4,8 @@ import { gzip } from '../compression/pako';
 import { BloomFilter, bloomFilterFromBytes } from '../data/bloom';
 
 const CREATE_PATH = '/create';
+const MODIFY_PATH = '/modify';
+const CLEAR_PATH = '/clear';
 const DESTROY_PATH = '/destroy';
 const CHECK_PATH = '/check';
 const DELETE_PATH = '/delete';
@@ -215,6 +217,18 @@ class Api {
     return await getData(this.apiKey, this.urlFor(bucketName, META_PATH), true);
   }
 
+  /**
+   * Modify a bucket's properties.
+   *
+   * @param bucketName The name of the bucket.
+   * @param dataJson A JSON-encoded string of the bucket metadata. Supports the same fields as `create()`.
+   * @returns Bucket metadata after update.
+   */
+  async modify(bucketName: string, dataJson: string): Promise<BucketMetadata> {
+    return await postData(this.apiKey, this.urlFor(bucketName, MODIFY_PATH), dataJson, true);
+  }
+
+
   /**
    * Get the Bloom filter for keys in this bucket. The Bloom filter contains all
    * keys ever inserted into this bucket; it does not remove deleted keys.
@@ -304,6 +318,16 @@ class Api {
     );
   }
 
+  /** Clear contents of this bucket. */
+  async clear(bucketName: string) {
+    await postData(
+      this.apiKey,
+      this.urlFor(bucketName, CLEAR_PATH),
+      '',
+      false
+    );
+  }
+
   /** Write to this bucket. */
   async write(bucketName: string, data: Uint8Array) {
     await postData(