[BLY-66] direct upload pparams (#28)

* bucket check and async setup
clients perform direct setup by default

* (python) more consistent json for internal api

all requests and response are JSON.
all binary payloads are explicitly encoded as base64
within api.py, and decoded back to bytes before leaving api.py.
User-facing code, e.g. bucket.py and bucket_service.py,
should not see base64 wrangling.

* Support async for all ops

refactor api.py to be async-first
use new asyncio loops to support non-async interface;
cannot call non-async methods from async context

* [js] update client to work with unified service
bump both versions to 0.2.1
disable npm/pypi publish except on manual workflow run

* disable request compression

* fix workflow tests

update standalone Spiral test server to use new JSON interface

Author: neilmovva

.github/workflows/build-js.yml

@@ -67,7 +67,7 @@ jobs:
   client-publish:
     name: Publish JavaScript SDK
     runs-on: ubuntu-latest
-    if: ${{ inputs.publish || github.ref == 'refs/heads/main' }}
+    if: ${{ inputs.publish }}
     needs: [client-build]
     steps:
       - uses: actions/checkout@v3

.github/workflows/build-python.yml

@@ -144,7 +144,7 @@ jobs:
   publish:
     name: Publish
     runs-on: ubuntu-latest
-    if: ${{ inputs.publish || github.ref == 'refs/heads/main' }}
+    if: ${{ inputs.publish }}
     needs: [linux, macos, windows, sdist]
     steps:
       - uses: actions/download-artifact@v3

e2e-tests/api.ts

@@ -30,15 +30,29 @@ function generateKeys(n: number, seed: number = 0): string[] {
     );
 }
 
+
+async function generateKVPairs(n: number, seed: number, itemSize: number): Promise<{ [key: string]: Uint8Array }> {
+    const keys = generateKeys(n, seed);
+    const kvPairs: { [key: string]: Uint8Array } = {};
+
+    for (const key of keys) {
+        kvPairs[key] = await keyToValue(key, itemSize);
+    }
+
+    return kvPairs;
+}
+
+
+function getRandomKey(kvP: { [key: string]: Uint8Array }): string {
+    return Object.keys(kvP)[Math.floor(Math.random() * Object.keys(kvP).length)];
+}
+
+
 function generateBucketName(): string {
     return 'api-tester-' + Math.random().toString(16).substring(2, 10);
 }
 
-async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev') {
-    const apiKey = process.env.BLYSS_API_KEY;
-    if (!apiKey) {
-        throw new Error('BLYSS_API_KEY environment variable is not set');
-    }
+async function testBlyssService(endpoint: string, apiKey: string) {
     console.log('Using key: ' + apiKey + ' to connect to ' + endpoint);
     const client: Client = await new blyss.Client(
         {
@@ -55,50 +69,44 @@ async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev')
     // generate N random keys
     const N = 100;
     const itemSize = 32;
-    let localKeys = generateKeys(N);
-    function getRandomKey(): string {
-        return localKeys[Math.floor(Math.random() * localKeys.length)];
-    }
+    let kvPairs = await generateKVPairs(N, 0, itemSize);
+
     // write all N keys
     await bucket.write(
-        await Promise.all(localKeys.map(
-            async (k) => ({
-                k: await keyToValue(k, itemSize)
-            })
-        ))
+        kvPairs
     );
     console.log(`Wrote ${N} keys`);
 
     // read a random key
-    let testKey = getRandomKey();
+    let testKey = getRandomKey(kvPairs);
+    console.log(`Reading key ${testKey}`)
+    await bucket.setup();
+    console.log("1111");
     let value = await bucket.privateRead(testKey);
     await verifyRead(testKey, value);
     console.log(`Read key ${testKey}`);
 
     // delete testKey from the bucket, and localData.
     await bucket.deleteKey(testKey);
-    localKeys.splice(localKeys.indexOf(testKey), 1);
+
     console.log(`Deleted key ${testKey}`);
 
     // write a new value
     testKey = 'newKey0';
-    await bucket.write({ testKey: keyToValue(testKey, itemSize) });
-    localKeys.push(testKey);
+    let newValue = await keyToValue(testKey, itemSize);
+    await bucket.write({ testKey: newValue });
+    kvPairs[testKey] = newValue;
     console.log(`Wrote key ${testKey}`);
 
     // clear all keys
     await bucket.clearEntireBucket();
-    localKeys = [];
+    kvPairs = {};
     console.log('Cleared bucket');
 
     // write a new set of N keys
-    localKeys = generateKeys(N, 1);
+    kvPairs = await generateKVPairs(N, 1, itemSize);
     await bucket.write(
-        await Promise.all(localKeys.map(
-            async (k) => ({
-                k: await keyToValue(k, itemSize)
-            })
-        ))
+        kvPairs
     );
     console.log(`Wrote ${N} keys`);
 
@@ -109,7 +117,7 @@ async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev')
     console.log(await bucket.info());
 
     // random read
-    testKey = getRandomKey();
+    testKey = getRandomKey(kvPairs);
     value = await bucket.privateRead(testKey);
     await verifyRead(testKey, value);
     console.log(`Read key ${testKey}`);
@@ -119,11 +127,15 @@ async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev')
     console.log(`Destroyed bucket ${bucket.name}`);
 }
 
-async function main() {
-    const endpoint = "https://dev2.api.blyss.dev"
-    console.log('Testing Blyss service at URL ' + endpoint);
-    await testBlyssService(endpoint);
+async function main(endpoint: string, apiKey: string) {
+    if (!apiKey) {
+        throw new Error('BLYSS_API_KEY environment variable is not set');
+    }
+    await testBlyssService(endpoint, apiKey);
     console.log('All tests completed successfully.');
 }
 
-main();
+// get endpoint and api key from command line, or fallback to defaults
+const endpoint = process.argv[2] || 'https://beta.api.blyss.dev';
+const apiKey = process.argv[3] || process.env.BLYSS_API_KEY;
+main(endpoint, apiKey);

e2e-tests/tests/simple.ts

@@ -8,17 +8,20 @@ export default async function main(port: string) {
 
   console.log(bucket.metadata);
 
+  // buckets are bytes-in/bytes-out. SDK write() will automatically serialize as UTF-8.
   await bucket.write({
     Ohio: 'Columbus',
     California: 'Sacramento'
   });
 
-  let capital = await bucket.privateRead('Ohio');
+  // but reads are always bytes-out, and must be decoded.
+  let capital = new TextDecoder().decode(await bucket.privateRead('Ohio'));
   if (capital !== 'Columbus') {
     throw 'Incorrect result.';
   }
 
-  capital = await bucket.privateRead('California');
+  // capital = await bucket.privateRead('California');
+  capital = new TextDecoder().decode(await bucket.privateRead('California'));
   if (capital !== 'Sacramento') {
     throw 'Incorrect result.';
   }

js/bridge/Cargo.lock

@@ -152,34 +152,27 @@ export class Bucket {
     keys: string[]
   ): Promise<any[]> {
     if (!this.uuid || !this.check(this.uuid)) {
+      console.log('Setting up client');
       await this.setup();
     }
 
-    const queries: { key: string; queryData: Uint8Array }[] = [];
+    // For each key, generate a query, encode it as base64, and append it to a list.
+    let queries: Uint8Array[] = [];
     for (const key of keys) {
       const rowIdx = this.lib.getRow(key);
       const queryData = this.lib.generateQuery(this.uuid, rowIdx);
-      queries.push({ key, queryData });
+      queries.push(queryData);
     }
 
-    const endResults = [];
-    const batches = Math.ceil(queries.length / this.batchSize);
-    for (let i = 0; i < batches; i++) {
-      const queriesForBatch = queries.slice(
-        i * this.batchSize,
-        (i + 1) * this.batchSize
-      );
-
-      const queryBatch = serializeChunks(queriesForBatch.map(x => x.queryData));
-      const rawResultChunks = await this.getRawResponse(queryBatch);
-      const rawResults = deserializeChunks(rawResultChunks);
+    // Send the list of queries to the server.
+    const rawResults = await this.api.privateReadJson(this.name, queries);
 
-      const batchEndResults = await Promise.all(
-        rawResults.map((r, i) => this.getEndResult(queriesForBatch[i].key, r))
-      );
 
-      endResults.push(...batchEndResults);
-    }
+    // For each query, decrypt the result, decompress it, and extract the
+    // result.
+    const endResults = await Promise.all(
+      rawResults.map((r, i) => this.getEndResult(keys[i], r))
+    );
 
     return endResults;
   }
@@ -328,12 +321,6 @@ export class Bucket {
     this.name = newBucketName;
   }
 
-  /** Gets info on all keys in this bucket. */
-  async listKeys(): Promise<KeyInfo[]> {
-    this.ensureSpiral();
-    return await this.api.listKeys(this.name);
-  }
-
   /**
    * Make a write to this bucket.
    *
@@ -343,27 +330,20 @@ export class Bucket {
    *   1024 UTF-8 bytes.
    */
   async write(
-    keyValuePairs: { [key: string]: any }
+    keyValuePairs: { [key: string]: Uint8Array | string | null }
   ) {
     this.ensureSpiral();
-
-    const data = [];
+    // convert any string KV pairs to Uint8Array
+    const kvPairs: { [key: string]: Uint8Array | null } = {};
     for (const key in keyValuePairs) {
-      if (Object.prototype.hasOwnProperty.call(keyValuePairs, key)) {
-        const value = keyValuePairs[key];
-        const valueBytes = serialize(value);
-        const keyBytes = new TextEncoder().encode(key);
-        const serializedKeyValue = wrapKeyValue(keyBytes, valueBytes);
-        data.push(serializedKeyValue);
-        // const kv = {
-        //   key: key,
-        //   value: Buffer.from(valueBytes).toString('base64')
-        // }
+      const value = keyValuePairs[key];
+      if (!(value instanceof Uint8Array)) {
+        kvPairs[key] = new TextEncoder().encode(value);
+      } else {
+        kvPairs[key] = value;
       }
     }
-    const concatenatedData = concatBytes(data);
-    // const concatenatedData = serialize(data);
-    await this.api.write(this.name, concatenatedData);
+    await this.api.write(this.name, kvPairs);
   }
 
   /**

js/bucket/bucket.ts

@@ -97,7 +97,7 @@ export class BucketService {
     const parameters = { ...DEFAULT_BUCKET_PARAMETERS, ...params };
     const bucketCreateReq = {
       name: bucketName,
-      parameters: JSON.stringify(parameters),
+      parameters,
       open_access: openAccess
     };
     await this.api.create(JSON.stringify(bucketCreateReq));