[offline] Support bootstrapping mirrored indices from local files/resources

Author: Clément Le Provost

algoliasearch/src/offline/java/com/algolia/search/saas/BootstrapListener.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2012-2016 Algolia
+ * http://www.algolia.com/
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.algolia.search.saas;
+
+import android.support.annotation.NonNull;
+
+/**
+ * Listener for events related to index bootstrapping.
+ *
+ * Notifications are sent on a per-index basis, but you may register the same listener for all indices.
+ * Notifications are sent on the main thread.
+ */
+public interface BootstrapListener
+{
+    /**
+     * Bootstrapping has just started.
+     *
+     * @param index The bootstrapping index.
+     */
+    public void bootstrapDidStart(@NonNull MirroredIndex index);
+
+    /**
+     * Bootstrapping has just finished.
+     *
+     * @param index The bootstrapping index.
+     * @param error Null if success, otherwise indicates the error.
+     */
+    public void bootstrapDidFinish(@NonNull MirroredIndex index, Throwable error);
+}

algoliasearch/src/offline/java/com/algolia/search/saas/FileUtils.java

@@ -26,6 +26,9 @@
 import android.support.annotation.NonNull;
 
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 
 /**
  * Various filesystem-related utilities.
@@ -49,4 +52,26 @@ public static boolean deleteRecursive(@NonNull File item)
         ok = ok && item.delete();
         return ok;
     }
+
+    /**
+     * Write a stream of bytes to a file.
+     *
+     * @param destinationFile The file to be written to. The parent directory must exist. If the file already exists,
+     *                        it will be overwritten.
+     * @param content The bytes to write.
+     * @throws IOException if anything goes wrong.
+     */
+    public static void writeFile(@NonNull File destinationFile, @NonNull InputStream content) throws IOException {
+        byte[] buffer = new byte[64 * 1024]; // 64 kB buffer
+        FileOutputStream outputStream = new FileOutputStream(destinationFile);
+        try {
+            int bytesRead;
+            while ((bytesRead = content.read(buffer)) >= 0) {
+                outputStream.write(buffer, 0, bytesRead);
+            }
+        } finally {
+            content.close();
+            outputStream.close();
+        }
+    }
 }

algoliasearch/src/offline/java/com/algolia/search/saas/MirroredIndex.java

@@ -23,7 +23,9 @@
 
 package com.algolia.search.saas;
 
+import android.content.res.Resources;
 import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
 import android.util.Log;
 
 import com.algolia.search.offline.core.LocalIndex;
@@ -35,6 +37,7 @@
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.io.Writer;
@@ -83,6 +86,53 @@
  * NOTE: The strategy applies both to {@link #searchAsync(Query, CompletionHandler)} and
  * {@link #searchDisjunctiveFacetingAsync(Query, List, Map, CompletionHandler)}.
  *
+ *
+ * ## Bootstrapping
+ *
+ * Before the first sync has successfully completed, a mirrored index is not available offline, because it has simply
+ * no data to search in yet. In most cases, this is not a problem: the app will sync as soon as possible, so unless
+ * the device is offline when the app is started for the first time, or unless search is required right after the
+ * first launch, the user should not notice anything.
+ *
+ * However, in some cases, you might need to have offline data available as soon as possible. To achieve that,
+ * `MirroredIndex` provides a **bootstrapping** feature.
+ *
+ * Bootstrapping consists in prepackaging with your app the data necessary to build your index, in JSON format;
+ * namely:
+ *
+ * - settings (one file)
+ * - objects (as many files as needed, each containing an array of objects)
+ *
+ * Then, upon application startup, call one of the `bootstrap*` methods. This will check if data already exists for the
+ * mirror and, if not, populate the mirror with the provided data. It also guarantees that a sync will not be started
+ * in the meantime, thus avoiding race conditions.
+ *
+ * ### Discussion
+ *
+ * **Warning:** We strongly advise against prepackaging index files. While it may work in some cases, Algolia Offline
+ * makes no guarantee whatsoever that the index file format will remain backward-compatible forever, nor that it
+ * is independent of the hardware architecture (e.g. 32 bits vs 64 bits, or Little Endian vs Big Endian). Instead,
+ * always use the official bootstrapping feature.
+ *
+ * While bootstrapping involves building the offline index on the device, and therefore incurs a small delay before
+ * the mirror is actually usable, using plain JSON offers a few advantages compared to prepackaging the index file
+ * itself:
+ *
+ * - You only need to ship the raw object data, which is smaller than shipping an entire index file, which contains
+ *   both the raw data *and* indexing metadata.
+ *
+ * - Plain JSON compresses well with standard compression techniques like GZip, whereas an index file has a binary
+ *   format which doesn't compress very efficiently.
+ *
+ * - Build automation is facilitated: you can easily extract the required data from your back-end, whereas building
+ *   an index would involve running the app on each mobile platform as part of your build process and capturing the
+ *   filesystem.
+ *
+ * Also, the build process is purposedly single-threaded across all indices, which means that on most modern devices
+ * with multi-core CPUs, the impact of bootstrapping on the app's performance will be very moderate, especially
+ * regarding UI responsiveness.
+ *
+ *
  * ## Limitations
  *
  * Algolia's core features are fully supported offline, including (but not limited to): **ranking**,
@@ -119,6 +169,7 @@ public class MirroredIndex extends Index
     private SyncStats stats;
 
     private Set<SyncListener> syncListeners = new HashSet<>();
+    private Set<BootstrapListener> bootstrapListeners = new HashSet<>();
 
     // ----------------------------------------------------------------------
     // Constants
@@ -560,6 +611,107 @@ public void run()
         }
     }
 
+    // ----------------------------------------------------------------------
+    // Bootstrapping
+    // ----------------------------------------------------------------------
+
+    /**
+     * Bootstrap the local mirror with local data stored on the filesystem.
+     *
+     * **Note:** This method will do nothing if offline data is already available, making it safe to call at every
+     * application launch.
+     *
+     * @param settingsFile Absolute path to the file containing the index settings, in JSON format.
+     * @param objectFiles Absolute path(s) to the file(s) containing the objects. Each file must contain an array of
+     *                    objects, in JSON format.
+     */
+    public void bootstrapFromFiles(@NonNull final File settingsFile, @NonNull final File... objectFiles) {
+        getClient().localBuildExecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                // Abort immediately if data already exists.
+                if (localIndex.exists()) {
+                    return;
+                }
+                _bootstrap(settingsFile, objectFiles);
+            }
+        });
+    }
+
+    /**
+     * Bootstrap the local mirror with local data stored in raw Android resources.
+     *
+     * **Note:** This method will do nothing if offline data is already available, making it safe to call at every
+     * application launch.
+     *
+     * @param resources A {@link Resources} instance to read resources from.
+     * @param settingsResId Resource identifier of the index settings, in JSON format.
+     * @param objectsResIds Resource identifiers of the various objects files. Each file must contain an array of
+     *                    objects, in JSON format.
+     */
+    public void bootstrapFromRawResources(@NonNull final Resources resources, @NonNull final int settingsResId, @NonNull final int... objectsResIds) {
+        getClient().localBuildExecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                // Abort immediately if data already exists.
+                if (localIndex.exists()) {
+                    return;
+                }
+                // Save resources to independent files on disk.
+                // TODO: See if we can have the Offline Core read directly from resources or assets.
+                File tmpDir = new File(getClient().getTempDir(), UUID.randomUUID().toString());
+                try {
+                    tmpDir.mkdirs();
+                    // Settings.
+                    File settingsFile = new File(tmpDir, "settings.json");
+                    FileUtils.writeFile(settingsFile, resources.openRawResource(settingsResId));
+                    // Objects.
+                    File[] objectFiles = new File[objectsResIds.length];
+                    for (int i = 0; i < objectsResIds.length; ++i) {
+                        objectFiles[i] = new File(tmpDir, "objects#" + Integer.toString(objectsResIds[i]) + ".json");
+                        FileUtils.writeFile(objectFiles[i], resources.openRawResource(objectsResIds[i]));
+                    }
+                    // Build the index.
+                    _bootstrap(settingsFile, objectFiles);
+                } catch (IOException e) {
+                    Log.e(MirroredIndex.class.getSimpleName(), "Failed to write bootstrap resources to disk", e);
+                } finally {
+                    // Delete temporary files.
+                    FileUtils.deleteRecursive(tmpDir);
+                }
+            }
+        });
+    }
+
+    private void _bootstrap(@NonNull File settingsFile, @NonNull File... objectFiles) {
+        // Notify listeners.
+        getClient().mainHandler.post(new Runnable() {
+            @Override
+            public void run() {
+                fireBootstrapDidStart();
+            }
+        });
+
+        // Build the index.
+        String[] objectFilePaths = new String[objectFiles.length];
+        for (int i = 0; i < objectFiles.length; ++i) {
+            objectFilePaths[i] = objectFiles[i].getAbsolutePath();
+        }
+        final int status = localIndex.build(settingsFile.getAbsolutePath(), objectFilePaths, true /* clearIndex */, null /* deletedObjectIDs */);
+
+        // Notify listeners.
+        getClient().mainHandler.post(new Runnable() {
+            @Override
+            public void run() {
+                Throwable error = null;
+                if (status != 200) {
+                    error = new AlgoliaException(String.format("Failed to bootstrap index \"%s\"", MirroredIndex.this.getIndexName()), status);
+                }
+                fireBootstrapDidFinish(error);
+            }
+        });
+    }
+
     // ----------------------------------------------------------------------
     // Search
     // ----------------------------------------------------------------------
@@ -1076,6 +1228,8 @@ private JSONObject _browseMirror(@NonNull Query query) throws AlgoliaException
     // Listeners
     // ----------------------------------------------------------------------
 
+    // SyncListener
+
     /**
      * Add a listener for sync events.
      * @param listener The listener to add.
@@ -1107,4 +1261,34 @@ private void fireSyncDidFinish()
             listener.syncDidFinish(this, error, stats);
         }
     }
+
+    // BootstrapListener
+
+    /**
+     * Add a listener for bootstrapping events.
+     * @param listener The listener to add.
+     */
+    public void addBootstrapListener(@NonNull BootstrapListener listener) {
+        bootstrapListeners.add(listener);
+    }
+
+    /**
+     * Remove a listener for bootstrapping events.
+     * @param listener The listener to remove.
+     */
+    public void removeBootstrapListener(@NonNull BootstrapListener listener) {
+        bootstrapListeners.remove(listener);
+    }
+
+    private void fireBootstrapDidStart() {
+        for (BootstrapListener listener : bootstrapListeners) {
+            listener.bootstrapDidStart(this);
+        }
+    }
+
+    private void fireBootstrapDidFinish(@Nullable Throwable error) {
+        for (BootstrapListener listener : bootstrapListeners) {
+            listener.bootstrapDidFinish(this, error);
+        }
+    }
 }