Merge pull request #910 from appwrite/feat-match-indexes

feat(cli): Pushing logic for indexes

Author: christyjacob4

templates/cli/lib/commands/push.js.twig

@@ -2,7 +2,7 @@ const chalk = require('chalk');
 const inquirer = require("inquirer");
 const JSONbig = require("json-bigint")({ storeAsString: false });
 const { Command } = require("commander");
-const { localConfig, globalConfig } = require("../config");
+const { localConfig, globalConfig, KeyAttributes } = require("../config");
 const { Spinner, SPINNER_ARC, SPINNER_DOTS } = require('../spinner');
 const { paginate } = require('../paginate');
 const { questionsPushBuckets, questionsPushTeams, questionsPushFunctions, questionsGetEntrypoint, questionsPushCollections, questionsConfirmPushCollections, questionsPushMessagingTopics, questionsPushResources } = require("../questions");
@@ -37,6 +37,7 @@ const {
     databasesUpdateRelationshipAttribute,
     databasesCreateRelationshipAttribute,
     databasesDeleteAttribute,
+    databasesDeleteIndex,
     databasesListAttributes,
     databasesListIndexes,
     databasesUpdateCollection
@@ -254,6 +255,41 @@ const awaitPools = {
             iteration + 1
         );
     },
+    deleteIndexes: async (databaseId, collectionId, indexesKeys, iteration = 1) => {
+        if (iteration > pollMaxDebounces) {
+            return false;
+        }
+
+        if (pollMaxDebounces === POLL_DEFAULT_VALUE) {
+            let steps = Math.max(1, Math.ceil(attributeKeys.length / STEP_SIZE));
+            if (steps > 1 && iteration === 1) {
+                pollMaxDebounces *= steps;
+
+                log('Found a large number of indexes to be deleted. Increasing timeout to ' + (pollMaxDebounces * POLL_DEBOUNCE / 1000 / 60) + ' minutes')
+            }
+        }
+
+        const { indexes } = await paginate(databasesListIndexes, {
+            databaseId,
+            collectionId,
+            parseOutput: false
+        }, 100, 'indexes');
+
+        const ready = indexesKeys.filter(index => indexes.includes(index.key));
+
+        if (ready.length === 0) {
+            return true;
+        }
+
+        await new Promise(resolve => setTimeout(resolve, POLL_DEBOUNCE));
+
+        return await awaitPools.expectIndexes(
+            databaseId,
+            collectionId,
+            indexesKeys,
+            iteration + 1
+        );
+    },
     expectIndexes: async (databaseId, collectionId, indexKeys, iteration = 1) => {
         if (iteration > pollMaxDebounces) {
             return false;
@@ -536,8 +572,18 @@ const updateAttribute = async (databaseId, collectionId, attribute) => {
             })
     }
 }
-const deleteAttribute = async (collection, attribute) => {
-    log(`Deleting attribute ${attribute.key} of ${collection.name} ( ${collection['$id']} )`);
+const deleteAttribute = async (collection, attribute, isIndex = false) => {
+    log(`Deleting ${isIndex ? 'index' : 'attribute'} ${attribute.key} of ${collection.name} ( ${collection['$id']} )`);
+
+    if (isIndex) {
+        await databasesDeleteIndex({
+            databaseId: collection['databaseId'],
+            collectionId: collection['$id'],
+            key: attribute.key,
+            parseOutput: false
+        });
+        return;
+    }
 
     await databasesDeleteAttribute({
         databaseId: collection['databaseId'],
@@ -568,6 +614,10 @@ const checkAttributeChanges = (remote, local, collection, recraeting = true) =>
     let attribute = remote;
 
     for (let key of Object.keys(remote)) {
+        if (!KeyAttributes.has(key)) {
+            continue;
+        }
+
         if (changeableKeys.includes(key)) {
             if (!recraeting) {
                 if (remote[key] !== local[key]) {
@@ -583,7 +633,12 @@ const checkAttributeChanges = (remote, local, collection, recraeting = true) =>
             continue;
         }
 
-        if (remote[key] !== local[key]) {
+        if (Array.isArray(remote[key]) && Array.isArray(local[key])) {
+            if (JSON.stringify(remote[key]) !== JSON.stringify(local[key])) {
+                const bol = reason === '' ? '' : '\n';
+                reason += `${bol}${key} changed from ${chalk.red(remote[key])} to ${chalk.green(local[key])}`;
+            }
+        } else if (remote[key] !== local[key]) {
             const bol = reason === '' ? '' : '\n';
             reason += `${bol}${key} changed from ${chalk.red(remote[key])} to ${chalk.green(local[key])}`;
         }
@@ -612,12 +667,9 @@ const generateChangesObject = (attribute, collection, isAdding) => {
 /**
  * Filter deleted and recreated attributes,
  * return list of attributes to create
- * @param remoteAttributes
- * @param localAttributes
- * @param collection
  * @returns {Promise<*|*[]>}
  */
-const attributesToCreate = async (remoteAttributes, localAttributes, collection) => {
+const attributesToCreate = async (remoteAttributes, localAttributes, collection, isIndex = false) => {
 
     const deleting = remoteAttributes.filter((attribute) => !attributesContains(attribute, localAttributes)).map((attr) => generateChangesObject(attr, collection, false));
     const adding = localAttributes.filter((attribute) => !attributesContains(attribute, remoteAttributes)).map((attr) => generateChangesObject(attr, collection, true));
@@ -639,10 +691,10 @@ const attributesToCreate = async (remoteAttributes, localAttributes, collection)
     }));
 
     if (!cliConfig.force) {
-        if (deleting.length > 0) {
+        if (deleting.length > 0 && !isIndex) {
             log(`Attribute deletion will cause ${chalk.red('loss of data')}`);
         }
-        if (conflicts.length > 0) {
+        if (conflicts.length > 0 && !isIndex) {
             log(`Attribute recreation will cause ${chalk.red('loss of data')}`);
         }
 
@@ -655,7 +707,7 @@ const attributesToCreate = async (remoteAttributes, localAttributes, collection)
 
     if (conflicts.length > 0) {
         changedAttributes = conflicts.map((change) => change.attribute);
-        await Promise.all(changedAttributes.map((changed) => deleteAttribute(collection, changed)));
+        await Promise.all(changedAttributes.map((changed) => deleteAttribute(collection, changed, isIndex)));
         remoteAttributes = remoteAttributes.filter((attribute) => !attributesContains(attribute, changedAttributes))
     }
 
@@ -665,7 +717,7 @@ const attributesToCreate = async (remoteAttributes, localAttributes, collection)
     }
 
     const deletingAttributes = deleting.map((change) => change.attribute);
-    await Promise.all(deletingAttributes.map((attribute) => deleteAttribute(collection, attribute)));
+    await Promise.all(deletingAttributes.map((attribute) => deleteAttribute(collection, attribute, isIndex)));
     const attributeKeys = [...remoteAttributes.map(attribute => attribute.key), ...deletingAttributes.map(attribute => attribute.key)]
 
     if (attributeKeys.length) {
@@ -1149,13 +1201,16 @@ const pushCollection = async ({ returnOnZero, attempts } = { returnOnZero: false
     // Serialize attribute actions
     for (let collection of collections) {
         let attributes = collection.attributes;
+        let indexes = collection.indexes;
 
         if (collection.isExisted) {
             attributes = await attributesToCreate(collection.remoteVersion.attributes, collection.attributes, collection);
+            indexes = await attributesToCreate(collection.remoteVersion.indexes, collection.indexes, collection, true);
 
-            if (Array.isArray(attributes) && attributes.length <= 0) {
+            if ((Array.isArray(attributes) && attributes.length <= 0) && (Array.isArray(indexes) && indexes.length <= 0)) {
                 continue;
             }
+
         }
 
         log(`Pushing collection ${collection.name} ( ${collection['databaseId']} - ${collection['$id']} ) attributes`)
@@ -1167,7 +1222,7 @@ const pushCollection = async ({ returnOnZero, attempts } = { returnOnZero: false
         }
 
         try {
-            await createIndexes(collection.indexes, collection);
+            await createIndexes(indexes, collection);
         } catch (e) {
             throw e;
         }

templates/cli/lib/config.js.twig

@@ -17,7 +17,7 @@ const KeyAttributes = new Set([
     "size",
     "default",
     // integer and float
-    "min", 
+    "min",
     "max",
     // email, enum, URL, IP, and datetime
     "format",
@@ -29,7 +29,10 @@ const KeyAttributes = new Set([
     "twoWay",
     "twoWayKey",
     "onDelete",
-    "side"
+    "side",
+    // Indexes
+    "attributes",
+    "orders"
 ]);
 const KeyIndexes = new Set(["key", "type", "status", "attributes", "orders"]);
 
@@ -604,4 +607,5 @@ class Global extends Config {
 module.exports = {
     localConfig: new Local(),
     globalConfig: new Global(),
+    KeyAttributes,
 };