feat: add data indexing API

lib/browser/client.js

@@ -105,6 +105,9 @@ merge(proto, require('../common/bucket/getBucketWebsite'));
 merge(proto, require('../common/bucket/putBucketWebsite'));
 merge(proto, require('../common/bucket/deleteBucketWebsite'));
 
+// bucket data index
+merge(proto, require('../common/bucket/dataIndex'));
+
 // lifecycle
 merge(proto, require('../common/bucket/getBucketLifecycle'));
 merge(proto, require('../common/bucket/putBucketLifecycle'));

lib/common/bucket/dataIndex.d.ts

@@ -0,0 +1,35 @@
+export declare function openMetaQuery(this: any, bucketName: string, options?: {}): Promise<{
+    res: any;
+    status: any;
+}>;
+export declare function getMetaQueryStatus(this: any, bucketName: string, options?: {}): Promise<{
+    res: any;
+    status: any;
+}>;
+interface ISubQuerie {
+    field?: string;
+    value?: string;
+    operation: string;
+    subQueries?: ISubQuerie[];
+}
+interface IAggregation {
+    field: string;
+    operation: string;
+}
+interface IMetaQuery {
+    nextToken?: string;
+    maxResults?: number;
+    query: ISubQuerie;
+    sort?: string;
+    order?: 'asc' | 'desc';
+    aggregations?: IAggregation[];
+}
+export declare function doMetaQuery(this: any, bucketName: string, queryParam: IMetaQuery, options?: {}): Promise<{
+    res: any;
+    status: any;
+}>;
+export declare function closeMetaQuery(this: any, bucketName: string, options?: {}): Promise<{
+    res: any;
+    status: any;
+}>;
+export {};

lib/common/bucket/dataIndex.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.closeMetaQuery = exports.doMetaQuery = exports.getMetaQueryStatus = exports.openMetaQuery = void 0;
+const checkBucketName_1 = require("../utils/checkBucketName");
+const obj2xml_1 = require("../utils/obj2xml");
+async function openMetaQuery(bucketName, options = {}) {
+    checkBucketName_1.checkBucketName(bucketName);
+    const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'add' }, options);
+    const result = await this.request(params);
+    return {
+        res: result.res,
+        status: result.status
+    };
+}
+exports.openMetaQuery = openMetaQuery;
+async function getMetaQueryStatus(bucketName, options = {}) {
+    checkBucketName_1.checkBucketName(bucketName);
+    const params = this._bucketRequestParams('GET', bucketName, 'metaQuery', options);
+    const result = await this.request(params);
+    return {
+        res: result.res,
+        status: result.status
+    };
+}
+exports.getMetaQueryStatus = getMetaQueryStatus;
+async function doMetaQuery(bucketName, queryParam, options = {}) {
+    checkBucketName_1.checkBucketName(bucketName);
+    const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'query' }, options);
+    const { aggregations } = queryParam;
+    let Aggregations;
+    if (aggregations && aggregations.length > 0) {
+        Aggregations = {
+            Aggregation: aggregations.map(item => ({ Field: item.field, Operation: item.operation }))
+        };
+    }
+    const paramXMLObj = {
+        MetaQuery: {
+            NextToken: queryParam.nextToken,
+            MaxResults: queryParam.maxResults,
+            Query: JSON.stringify(queryParam.query),
+            Sort: queryParam.sort,
+            Order: queryParam.order,
+            Aggregations
+        }
+    };
+    params.mime = 'xml';
+    params.content = obj2xml_1.obj2xml(paramXMLObj);
+    const result = await this.request(params);
+    return {
+        res: result.res,
+        status: result.status
+    };
+}
+exports.doMetaQuery = doMetaQuery;
+async function closeMetaQuery(bucketName, options = {}) {
+    checkBucketName_1.checkBucketName(bucketName);
+    const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'delete' }, options);
+    const result = await this.request(params);
+    return {
+        res: result.res,
+        status: result.status
+    };
+}
+exports.closeMetaQuery = closeMetaQuery;

lib/common/bucket/dataIndex.ts

@@ -0,0 +1,91 @@
+// https://help.aliyun.com/zh/oss/developer-reference/data-indexing
+import { checkBucketName } from '../utils/checkBucketName';
+import { obj2xml } from '../utils/obj2xml';
+
+export async function openMetaQuery(this: any, bucketName: string, options = {}) {
+  checkBucketName(bucketName);
+  const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'add' }, options);
+
+  const result = await this.request(params);
+  return {
+    res: result.res,
+    status: result.status
+  };
+}
+
+export async function getMetaQueryStatus(this: any, bucketName: string, options = {}) {
+  checkBucketName(bucketName);
+  const params = this._bucketRequestParams('GET', bucketName, 'metaQuery', options);
+
+  const result = await this.request(params);
+  return {
+    res: result.res,
+    status: result.status
+  };
+}
+
+interface ISubQuerie {
+  // https://help.aliyun.com/zh/oss/developer-reference/appendix-supported-fields-and-operators
+  field?: string;
+  value?: string;
+  // 操作符。取值范围为eq（等于）、gt（大于）、gte（大于等于）、lt（小于）、 lte（小于等于）、match（模糊查询）、prefix（前缀查询）、and（逻辑与）、or（逻辑或）和not（逻辑非）。
+  operation: string;
+  subQueries?: ISubQuerie[];
+}
+
+interface IAggregation {
+  field: string;
+  operation: string;
+}
+
+interface IMetaQuery {
+  nextToken?: string;
+  maxResults?: number;
+  query: ISubQuerie;
+  sort?: string;
+  order?: 'asc' | 'desc';
+  aggregations?: IAggregation[];
+}
+
+export async function doMetaQuery(this: any, bucketName: string, queryParam: IMetaQuery, options = {}) {
+  checkBucketName(bucketName);
+  const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'query' }, options);
+
+  const { aggregations } = queryParam;
+  let Aggregations;
+  if (aggregations && aggregations.length > 0) {
+    Aggregations = {
+      Aggregation: aggregations.map(item => ({ Field: item.field, Operation: item.operation }))
+    };
+  }
+
+  const paramXMLObj = {
+    MetaQuery: {
+      NextToken: queryParam.nextToken,
+      MaxResults: queryParam.maxResults,
+      Query: JSON.stringify(queryParam.query),
+      Sort: queryParam.sort,
+      Order: queryParam.order,
+      Aggregations
+    }
+  };
+  params.mime = 'xml';
+  params.content = obj2xml(paramXMLObj);
+
+  const result = await this.request(params);
+  return {
+    res: result.res,
+    status: result.status
+  };
+}
+
+export async function closeMetaQuery(this: any, bucketName: string, options = {}) {
+  checkBucketName(bucketName);
+  const params = this._bucketRequestParams('POST', bucketName, { metaQuery: '', comp: 'delete' }, options);
+
+  const result = await this.request(params);
+  return {
+    res: result.res,
+    status: result.status
+  };
+}

lib/common/bucket/index.js

@@ -32,3 +32,4 @@ merge(proto, require('./extendBucketWorm'));
 merge(proto, require('./getBucketWorm'));
 merge(proto, require('./initiateBucketWorm'));
 merge(proto, require('./getBucketStat'));
+merge(proto, require('./dataIndex'));

test/browser/browser.test.js

@@ -38,6 +38,7 @@ const cleanBucket = async store => {
   const uploads = result.uploads || [];
   await Promise.all(uploads.map(_ => store.abortMultipartUpload(_.name, _.uploadId)));
 };
+
 describe('browser', () => {
   /* eslint require-yield: [0] */
   before(() => {
@@ -2509,4 +2510,26 @@ describe('browser', () => {
       }
     });
   });
+
+  // oss server 不支持跨域 大概要到9月30号才上生产支持 跨域，后端是：云仁
+  describe.skip('bucket data index', () => {
+    let store;
+    before(async () => {
+      store = oss({ ...ossConfig, refreshSTSTokenInterval: 1000 });
+    });
+
+    it('open meta query of bucket', async () => {
+      try {
+        // await store.listV2({ 'max-keys': 1 });
+        // const result = await store.getMetaQueryStatus(stsConfig.bucket); // oss server does not support cross domain
+        const result = await store.openMetaQuery(stsConfig.bucket);
+        console.log('rr', result);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        console.log('bb', JSON.stringify(error), error);
+        assert.fail(error);
+      }
+    });
+  });
 });

test/node/bucket.test.js

@@ -1480,4 +1480,63 @@ describe('test/bucket.test.js', () => {
       });
     });
   });
+
+  describe('openMetaQuery() openMetaQuery() doMetaQuery() closeMetaQuery()', () => {
+    it('open meta query of bucket', async () => {
+      try {
+        const result = await store.openMetaQuery(bucket);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        assert.fail(error);
+      }
+    });
+
+    it('getMetaQueryStatus()', async () => {
+      try {
+        const result = await store.getMetaQueryStatus(bucket);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        assert.fail(error);
+      }
+    });
+
+    it('doMetaQuery()', async () => {
+      try {
+        const queryParam = {
+          query: { operation: 'and', subQueries: [{ Field: 'Size', Value: '1048576', Operation: 'lt' }] }
+        };
+        const result = await store.doMetaQuery(bucket, queryParam);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        assert.fail(error);
+      }
+    });
+
+    it('doMetaQuery() Aggregations', async () => {
+      try {
+        const queryParam = {
+          query: { operation: 'and', subQueries: [{ Field: 'Size', Value: '1048576', Operation: 'lt' }] },
+          aggregations: [{ field: 'Size', operation: 'sum' }]
+        };
+        const result = await store.doMetaQuery(bucket, queryParam);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        assert.fail(error);
+      }
+    });
+
+    it('closeMetaQuery()', async () => {
+      try {
+        const result = await store.closeMetaQuery(bucket);
+        assert.strictEqual(result.status, 200);
+        assert.deepEqual(result.res.statusMessage, 'OK');
+      } catch (error) {
+        assert.fail(error);
+      }
+    });
+  });
 });