Merge #427

427: Add populateEntryRule in plugin's settings r=bidoubiwa a=bidoubiwa

Fixes: #423
Fixes: #425

By default when fetching an entry from the Strapi database it will not always fetch every nested relationship. 
Imagine I have a collection that has a relationship with another collection, that itself has another relationship etc.. 
when fetching with `findOne` for example it will not go as deep. 

this PR introduces a setting that lets the user give the specific rule to provide to the fetching method to ensure that every nested relation is fetched how the user would like it. 

Co-authored-by: Charlotte Vermandel <[email protected]>
Co-authored-by: cvermand <[email protected]>

Author: meili-bors[bot]

README.md

@@ -350,6 +350,60 @@ module.exports = {
 
 [See resources](./resources/meilisearch-settings) for more settings examples.
 
+#### 👥 Populate entry rule
+
+Content-types in Strapi may have relationships with other content-types (ex: `restaurant` can have a many-to-many relation with `category`). To ensure that these links are fetched and added to an entry correctly from your Strapi database, the correct populate rule must be provided ([see documentation](https://docs-next.strapi.io/developer-docs/latest/developer-resources/database-apis-reference/entity-service/populate.html#basic-populating)).
+
+To communicate the populate rule, use the `populateEntryRule` setting on the according content-type in the plugin's settings.
+
+**For example**
+
+Imagine my `restaurant` content-type has a relation with a repeatable-component `repeatableComponent` that itself has a relationship with the content-type `categories`.
+
+The following population will ensure that a `restaurant` entry contains even the most nested relation.
+
+```js
+module.exports = {
+  meilisearch: {
+    config: {
+      restaurant: {
+        populateEntryRule: ['repeatableComponent.categories', 'categories'],
+      }
+    }
+  },
+}
+```
+
+by providing this, the following is indexed in Meilisearch:
+
+```json
+  {
+    "id": "restaurant-1",
+    "title": "The slimmy snail",
+    // ... other restaurant fields
+    "repeatableComponent": [
+      {
+        "id": 1,
+        "title": "my repeatable component 1"
+        "categories": [
+          {
+            "id": 3,
+            "name": "Asian",
+            // ... other category fields
+          },
+          {
+            "id": 2,
+            "name": "Healthy",
+            // ... other category fields
+          }
+        ],
+
+      }
+    ],
+
+  }
+```
+
 ### 🕵️‍♀️ Start Searching <!-- omit in toc -->
 
 Once you have a content-type indexed in Meilisearch, you can [start searching](https://docs.meilisearch.com/learn/getting_started/quick_start.html#search).

server/__tests__/configuration-validation.test.js

@@ -195,11 +195,11 @@ describe('Test plugin configuration', () => {
   test('Test settings with function', async () => {
     validateConfiguration({
       restaurant: {
-        settings: {},
+        settings: () => {},
       },
     })
     expect(fakeStrapi.log.warn).toHaveBeenCalledTimes(0)
-    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(0)
+    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(1)
   })
 
   test('Test settings with undefined', async () => {
@@ -212,6 +212,52 @@ describe('Test plugin configuration', () => {
     expect(fakeStrapi.log.error).toHaveBeenCalledTimes(0)
   })
 
+  test('Test populateEntryRule with wrong type', async () => {
+    validateConfiguration({
+      restaurant: {
+        populateEntryRule: 0,
+      },
+    })
+    expect(fakeStrapi.log.warn).toHaveBeenCalledTimes(0)
+    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(1)
+    expect(fakeStrapi.log.error).toHaveBeenCalledWith(
+      'the "populateEntryRule" param of "restaurant" should be an object/array/string'
+    )
+  })
+
+  test('Test populateEntryRule with function', async () => {
+    validateConfiguration({
+      restaurant: {
+        populateEntryRule: () => {},
+      },
+    })
+    expect(fakeStrapi.log.warn).toHaveBeenCalledTimes(0)
+    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(1)
+    expect(fakeStrapi.log.error).toHaveBeenCalledWith(
+      'the "populateEntryRule" param of "restaurant" should be an object/array/string'
+    )
+  })
+
+  test('Test populateEntryRule with empty object', async () => {
+    validateConfiguration({
+      restaurant: {
+        populateEntryRule: {},
+      },
+    })
+    expect(fakeStrapi.log.warn).toHaveBeenCalledTimes(0)
+    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(0)
+  })
+
+  test('Test populateEntryRule with undefined', async () => {
+    validateConfiguration({
+      restaurant: {
+        populateEntryRule: undefined,
+      },
+    })
+    expect(fakeStrapi.log.warn).toHaveBeenCalledTimes(0)
+    expect(fakeStrapi.log.error).toHaveBeenCalledTimes(0)
+  })
+
   test('Test configuration with random field ', async () => {
     validateConfiguration({
       restaurant: {

server/configuration-validation.js

@@ -47,6 +47,7 @@ function validateCollectionConfiguration({ configuration, collection }) {
     'transformEntry',
     'settings',
     'filterEntry',
+    'populateEntryRule',
   ]
 
   if (configuration === undefined) {
@@ -98,6 +99,18 @@ function validateCollectionConfiguration({ configuration, collection }) {
     delete configuration.settings
   }
 
+  if (
+    configuration.populateEntryRule !== undefined &&
+    !isObject(configuration.populateEntryRule) &&
+    !Array.isArray(configuration.populateEntryRule) &&
+    typeof configuration.populateEntryRule !== 'string'
+  ) {
+    strapi.log.error(
+      `the "populateEntryRule" param of "${collection}" should be an object/array/string`
+    )
+    delete configuration.populateEntryRule
+  }
+
   Object.keys(configuration).forEach(attribute => {
     if (!validApiFields.includes(attribute)) {
       strapi.log.warn(

server/services/content-types/content-types.js

@@ -197,11 +197,16 @@ module.exports = ({ strapi }) => ({
    *
    * @param  {object} options
    * @param  {string} options.contentType - Name of the content type.
+   * @param  {object} [options.populate] - Relations, components and dynamic zones to populate.
    * @param  {function} options.callback - Function applied on each entry of the contentType.
    *
    * @returns {Promise<any[]>} - List of all the returned elements from the callback.
    */
-  actionInBatches: async function ({ contentType, callback = () => {} }) {
+  actionInBatches: async function ({
+    contentType,
+    callback = () => {},
+    populate = '*',
+  }) {
     const BATCH_SIZE = 500
 
     // Need total number of entries in contentType
@@ -215,6 +220,7 @@ module.exports = ({ strapi }) => ({
           start: index,
           limit: BATCH_SIZE,
           contentType,
+          populate,
         })) || []
 
       const info = await callback({ entries, contentType })

server/services/lifecycle/lifecycle.js

@@ -25,24 +25,24 @@ module.exports = ({ strapi }) => {
 
           // Fetch complete entry instead of using result that is possibly
           // partial.
-          const fullEntry = await strapi.entityService.findOne(
-            contentTypeUid,
-            result.id,
-            { populate: '*' }
-          )
+          const entry = await contentTypeService.getEntry({
+            contentType: contentTypeUid,
+            id: result.id,
+            populate: meilisearch.populateEntryRule({ contentType }),
+          })
 
           meilisearch
             .addEntriesToMeilisearch({
               contentType: contentTypeUid,
-              entries: [fullEntry],
+              entries: [entry],
             })
             .catch(e => {
               strapi.log.error(
                 `Meilisearch could not add entry with id: ${result.id}: ${e.message}`
               )
             })
         },
-        afterCreateMany() {
+        async afterCreateMany() {
           strapi.log.error(
             `Meilisearch does not work with \`afterCreateMany\` hook as the entries are provided without their id`
           )
@@ -55,29 +55,29 @@ module.exports = ({ strapi }) => {
 
           // Fetch complete entry instead of using result that is possibly
           // partial.
-          const fullEntry = await strapi.entityService.findOne(
-            contentTypeUid,
-            result.id,
-            { populate: '*' }
-          )
+          const entry = await contentTypeService.getEntry({
+            contentType: contentTypeUid,
+            id: result.id,
+            populate: meilisearch.populateEntryRule({ contentType }),
+          })
 
           meilisearch
             .updateEntriesInMeilisearch({
               contentType: contentTypeUid,
-              entries: [fullEntry],
+              entries: [entry],
             })
             .catch(e => {
               strapi.log.error(
                 `Meilisearch could not update entry with id: ${result.id}: ${e.message}`
               )
             })
         },
-        afterUpdateMany() {
+        async afterUpdateMany() {
           strapi.log.error(
             `Meilisearch could not find an example on how to access the \`afterUpdateMany\` hook. Please consider making an issue to explain your use case`
           )
         },
-        afterDelete(event) {
+        async afterDelete(event) {
           const { result, params } = event
           const meilisearch = strapi
             .plugin('meilisearch')
@@ -106,7 +106,7 @@ module.exports = ({ strapi }) => {
               )
             })
         },
-        afterDeleteMany() {
+        async afterDeleteMany() {
           strapi.log.error(
             `Meilisearch could not find an example on how to access the \`afterDeleteMany\` hook. Please consider making an issue to explain your use case`
           )

server/services/meilisearch/config.js

@@ -23,7 +23,8 @@ module.exports = ({ strapi }) => {
     /**
      * Get the name of the index from Meilisearch in which the contentType content is added.
      *
-     * @param contentType - Name of the contentType.
+     * @param {object} options
+     * @param {string} options.contentType - ContentType name.
      *
      * @return {String} - Index name
      */
@@ -34,6 +35,21 @@ module.exports = ({ strapi }) => {
       return contentTypeConfig.indexName || collection
     },
 
+    /**
+     * Get the populate rule of a content-type that is applied when fetching entries in the Strapi database.
+     *
+     * @param {object} options
+     * @param {string} options.contentType - ContentType name.
+     *
+     * @return {String} - Populate rule.
+     */
+    populateEntryRule: function ({ contentType }) {
+      const collection = contentTypeService.getCollectionName({ contentType })
+      const contentTypeConfig = meilisearchConfig[collection] || {}
+
+      return contentTypeConfig.populate || '*'
+    },
+
     /**
      * Transform contentTypes entries before indexation in Meilisearch.
      *
@@ -83,7 +99,7 @@ module.exports = ({ strapi }) => {
      * @param {Array<Object>} options.entries  - The data to convert. Conversion will use
      * the static method `toSearchIndex` defined in the model definition
      *
-     * @return {Array<Object>} - Converted or mapped data
+     * @return {Promise<Array<Object>>} - Converted or mapped data
      */
     filterEntries: async function ({ contentType, entries = [] }) {
       const collection = contentTypeService.getCollectionName({ contentType })

server/services/meilisearch/connector.js

@@ -286,6 +286,7 @@ module.exports = ({ strapi, adapter, config }) => {
       const tasksUids = await contentTypeService.actionInBatches({
         contentType,
         callback: addDocuments,
+        populate: config.populateEntryRule({ contentType }),
       })
 
       await store.addIndexedContentType({ contentType })
@@ -344,6 +345,7 @@ module.exports = ({ strapi, adapter, config }) => {
         await contentTypeService.actionInBatches({
           contentType,
           callback: deleteEntries,
+          populate: config.populateEntryRule({ contentType }),
         })
       } else {
         const { apiKey, host } = await store.getCredentials()