feat: index only visible_to_public articles to algolia (#3001)

* feat: index only visible_to_public articles to algolia

* refactor: move isVisibleToPublic to article model

* chore: improvements on indexing

* spec: updated tests

Author: gkartalis

scripts/reindex_articles.coffee

@@ -3,8 +3,10 @@ mongojs = require 'mongojs'
 path = require 'path'
 { indexForSearch, indexForAlgolia } = Save = require '../src/api/apps/articles/model/distribute'
 Article = require '../src/api/apps/articles/model/index.js'
+ArticleModel = require '../src/api/models/article.coffee'
 search = require '../src/api/lib/elasticsearch'
 async = require 'async'
+{ cloneDeep } = require 'lodash'
 
 # Setup environment variables
 env = require 'node-env-file'
@@ -28,9 +30,12 @@ indexWorker = (article, cb) ->
   articlePresent = Article.present(article) 
   indexForSearch articlePresent, () =>
     console.log('indexed on Elasticsearch ', article.id or article._id)
-    if (articlePresent.published or articlePresent.scheduled_publish_at) and articlePresent.indexable
+    isArticleVisibleToPublic = new ArticleModel(cloneDeep article).isVisibleToPublic()
+    if (articlePresent.indexable and isArticleVisibleToPublic)
       indexForAlgolia articlePresent, () =>
         console.log('indexed on Algolia ', article.id or article._id)
         cb()
     else
-      cb()
+      removeFromAlgolia articlePresent, () =>
+        console.log('removed from Algolia ', article.id or article._id)
+        cb()

src/api/apps/articles/model/distribute.coffee

@@ -97,7 +97,6 @@ moment = require 'moment'
       section.body
   tags = article.tags
   tags = tags.concat article.vertical.name if article.vertical
-
   search.client.index(
     index: search.index,
     type: 'article',
@@ -109,7 +108,7 @@ moment = require 'moment'
       published: article.published
       published_at: article.published_at
       scheduled_publish_at: article.scheduled_publish_at
-      visible_to_public: article.published and sections?.length > 0 and article.channel_id?.toString() is EDITORIAL_CHANNEL
+      visible_to_public: new Article(cloneDeep article).isVisibleToPublic()
       author: article.author and article.author.name or ''
       featured: article.featured
       tags: tags
@@ -122,9 +121,6 @@ moment = require 'moment'
   )
 
 @indexForAlgolia = (article, cb) ->
-  if article.sections
-    sections = for section in article.sections
-      section.body
   tags = article.tags
   tags = tags.concat article.vertical.name if article.vertical
 

src/api/apps/articles/model/index.js

@@ -341,15 +341,18 @@ export const backfill = callback => {
       articles,
       (article, cb) => {
         console.log("---------------------")
-        console.log("---------------------")  
+        console.log("---------------------")
         console.log("---------------------")
         console.log("---------------------")
         console.log("---------------------")
         console.log("---------------------")
         console.log(
-          `Backfilling article: ${articles.indexOf(article) + 1} of ${articles.length}, ${article.slugs[article.slugs.length - 1]}, published at ${article.published_at}`
+          `Backfilling article: ${articles.indexOf(article) + 1} of ${
+            articles.length
+          }, ${article.slugs[article.slugs.length - 1]}, published at ${
+            article.published_at
+          }`
         )
-     
 
         /*
         Write backfill logic here. Make sure to callback with cb()

src/api/apps/articles/model/save.coffee

@@ -10,11 +10,13 @@ url = require 'url'
 Q = require 'bluebird-q'
 request = require 'superagent'
 Article = require './index'
+ArticleModel = require './../../../../api/models/article.coffee'
 { distributeArticle, deleteArticleFromSailthru, getArticleUrl, indexForSearch, indexForAlgolia, removeFromAlgolia } = require './distribute'
 { ARTSY_URL, GEMINI_CLOUDFRONT_URL } = process.env
 artsyXapp = require('artsy-xapp')
 { sanitizeLink } = require "./sanitize.js"
 chalk = require 'chalk'
+{ cloneDeep } = require 'lodash'
 
 @onPublish = (article, cb) =>
   unless article.published_at or article.scheduled_publish_at
@@ -163,14 +165,18 @@ removeStopWords = (title) ->
   if article.published or article.scheduled_publish_at
     article = setOnPublishFields article
     indexForSearch(article, ->) if article.indexable
-    indexForAlgolia(article, ->) if article.indexable
     distributeArticle article, =>
       db.articles.save sanitize(article), callback
   else
     indexForSearch(article, ->) if article.indexable
-    removeFromAlgolia(article.id, ->) if article.indexable
     db.articles.save sanitize(article), callback
 
+  isArticleVisibleToPublic = new ArticleModel(cloneDeep article).isVisibleToPublic()
+  if (isArticleVisibleToPublic and article.indexable)
+    indexForAlgolia(article, ->)
+  else
+    removeFromAlgolia(article.id, ->)
+
 # TODO: Create a Joi plugin for this https://github.com/hapijs/joi/issues/577
 sanitize = (article) ->
   if article.sections

src/api/apps/articles/test/model/save.spec.ts

@@ -615,7 +615,6 @@ describe("Save", () => {
           if (err) {
             done(err)
           }
-          indexForAlgolia.callCount.should.eql(1)
           indexForSearch.callCount.should.eql(1)
           done()
         })
@@ -643,7 +642,7 @@ describe("Save", () => {
 
     it("indexes published & indexable articles to algolia", done => {
       Save.sanitizeAndSave(() => {
-        Article.find("5086df098523e60002000011", (err, _article) => {
+        Article.find("54276766fd4f50996aeca2b8", (err, _article) => {
           if (err) {
             done(err)
           }
@@ -653,7 +652,14 @@ describe("Save", () => {
       })(null, {
         indexable: true,
         published: true,
-        _id: ObjectId("5086df098523e60002000011"),
+        _id: ObjectId("54276766fd4f50996aeca2b8"),
+        sections: [
+          {
+            type: "text",
+            body: "<a href='http://artsy.net/artist/andy-warhol'>link</a>",
+          },
+        ],
+        channel_id: "5086df098523e60002000018",
       })
     })
 

src/api/models/article.coffee

@@ -41,6 +41,9 @@ module.exports = class Article extends Backbone.Model
   isEditorial: ->
     @get('channel_id')?.toString() is EDITORIAL_CHANNEL
 
+  isVisibleToPublic: ->
+    @get('published') && @isEditorial() && @get('sections')?.length > 0
+
   replaceTagWith: (htmlStr, findTag, replaceTag) ->
     $ = cheerio.load(htmlStr)
     $(findTag).each ->