Introduce ContentModel2/Collections (WIP)

Start of a major change in content modelling. Develop inside master without touching the working version.

Goal:

- The old contentModel structure becomes a "collection" of "entries".
- Root possibly has many collections.
- Each collection will be of a certain content type. e.g. TextEntry
  Collection
- Collections have categorization and tagging within them
- stuff like: example.com/blog, example.com/projects,
  example.com/podcast (each directory is a collection)

Notes:

- Bypassed bringing date from git during enhancements
- Eliminated everything async from contentModel(2) (for clarity and testability)
- Removed outputPath from categories and posts because it caused issues
  and also seemed it shouldn't be a concern of contentModel to calculate
  outputPath (we'll see).
- Identified posts and categories by their fsObject.path
- Removed subpages and homepage from linking enhancer
- Allowed contentModel hook to be run for each collection
- Computed entry and tag permalinks based on fsObject.path (feels like the containing
  collection should be available to entry scopes)

Author: scriptype

src/compiler/contentModel2/collection/contentTypes.js

@@ -0,0 +1,7 @@
+module.exports = {
+  COLLECTION: 'collection',
+  CATEGORY: 'category',
+  CATEGORY_INDEX: 'categoryIndex',
+  POST: 'post',
+  FOLDERED_POST_INDEX: 'folderedPostIndex'
+}

src/compiler/contentModel2/collection/enhancers/dates.js

@@ -0,0 +1,32 @@
+const _ = require('lodash')
+const Dictionary = require('../../../../dictionary')
+const { decorate } = require('../../../../decorations')
+
+const withDates = (entry) => {
+  if (!entry.publishDatePrototype) {
+    return entry
+  }
+  const locale = Dictionary.getLocale()
+  const decoratedEntry = entry //await decorate('publishDate', entry)
+  const publishDate = new Date(decoratedEntry.publishDatePrototype.value)
+  return {
+    ..._.omit(entry, 'publishDatePrototype'),
+    publishDate,
+    publishDateUTC: publishDate.toUTCString(),
+    publishDateFull: publishDate.toLocaleString(locale, { dateStyle: 'full' }),
+    publishDateLong: publishDate.toLocaleString(locale, { dateStyle: 'long' }),
+    publishDateMedium: publishDate.toLocaleString(locale, { dateStyle: 'medium' }),
+    publishDateShort: publishDate.toLocaleString(locale, { dateStyle: 'short' })
+  }
+}
+
+module.exports = (contentModel) => {
+  return {
+    ...contentModel,
+    categories: contentModel.categories.map(category => ({
+      ...category,
+      posts: category.posts.map(withDates)
+    })),
+    posts: contentModel.posts.map(withDates)
+  }
+}

src/compiler/contentModel2/collection/enhancers/links.js

@@ -0,0 +1,185 @@
+const { sep } = require('path')
+const _ = require('lodash')
+const Settings = require('../../../../settings')
+const { pipeSync, getSlug, makePermalink } = require('../../../../helpers')
+
+const getScoreByOverlap = (arrayOfStrings1 = [], arrayOfStrings2 = []) => {
+  const overlap = _.intersection(
+    arrayOfStrings1.map(string => string.toUpperCase()),
+    arrayOfStrings2.map(string => string.toUpperCase())
+  ).filter(Boolean)
+  return overlap.length
+}
+
+const getScoreByEquality = (thing1, thing2) => {
+  return Number(thing1 === thing2)
+}
+
+const getScoreByTextSimilarity = (text1, text2) => {
+  return getScoreByOverlap(text1.split(' '), text2.split(' '))
+}
+
+const attachPaging = (post, postIndex, posts) => {
+  const paging = {}
+  if (postIndex > 0) {
+    paging.nextPost = {
+      title: posts[postIndex - 1].title,
+      permalink: posts[postIndex - 1].permalink
+    }
+  }
+  if (postIndex < posts.length - 1) {
+    paging.previousPost = {
+      title: posts[postIndex + 1].title,
+      permalink: posts[postIndex + 1].permalink
+    }
+  }
+  return paging
+}
+
+const normalizeText = (text = '') => {
+  return text
+    .replace(/<(p|a|img|pre|video|i|u|em|strong|small|div|section)>/i, '')
+    .replace(/-_:'\(\)\[\]\{\}\./g, '')
+    .replace(/\n/g, '')
+}
+
+const attachRelevantPosts = (post, posts) => {
+  const relevantPosts = []
+  posts.forEach(otherPost => {
+    if (otherPost.permalink === post.permalink) {
+      return
+    }
+    const tagsOverlapScore = getScoreByOverlap(
+      post.tags.map(t => t.tag),
+      otherPost.tags.map(t => t.tag)
+    )
+    const titleSimilarityScore = getScoreByTextSimilarity(
+      normalizeText(post.title),
+      normalizeText(otherPost.title)
+    )
+    const sameCategoryScore = getScoreByEquality(post.category, otherPost.category)
+    const relevancyScore = (
+      tagsOverlapScore +
+      titleSimilarityScore +
+      sameCategoryScore * 3
+    )
+    if (relevancyScore > 0) {
+      const relevantPost = {
+        ..._.pick(otherPost, [
+          'title',
+          'permalink',
+          'category'
+        ]),
+        relevancyScore
+      }
+      relevantPosts.push(relevantPost)
+    }
+  })
+  return {
+    relevantPosts: relevantPosts.sort(
+      (a, b) => b.relevancyScore - a.relevancyScore
+    )
+  }
+}
+
+const linkPosts = (contentModel) => {
+  return {
+    ...contentModel,
+    posts: contentModel.posts.map((post) => {
+      const category = contentModel.categories.find(cat => {
+        return cat.path === post.category.path
+      })
+      const postInCategory = category.posts.find(p => p.path === post.path)
+      const postIndex = category.posts.indexOf(postInCategory)
+      return {
+        ...post,
+        links: {
+          ...(post.links || {}),
+          ...attachPaging(post, postIndex, category.posts),
+          ...attachRelevantPosts(post, category.posts)
+        }
+      }
+    })
+  }
+}
+
+const attachMentionedEntries = (allEntries) => (entry) => {
+  const mention = (contentModelEntry) => ({
+    title: contentModelEntry.title || contentModelEntry.name,
+    permalink: contentModelEntry.permalink,
+    category: contentModelEntry.category
+  })
+
+  const otherEntries = allEntries.filter(otherEntry => {
+    return otherEntry.permalink !== entry.permalink
+  })
+
+  const entriesMentioned = otherEntries
+    .filter(otherEntry => entry.mentions.includes(otherEntry.permalink))
+    .map(mention)
+
+  const entriesMentionedBy = otherEntries
+    .filter(otherEntry => otherEntry.mentions.includes(entry.permalink))
+    .map(mention)
+
+  return {
+    ..._.omit(entry, 'mentions'),
+    links: {
+      ...(entry.links || {}),
+      mentionedTo: entriesMentioned,
+      mentionedBy: entriesMentionedBy
+    }
+  }
+}
+
+const linkMentionedEntries = (contentModel) => {
+  const attacher = attachMentionedEntries([
+    ...contentModel.posts,
+    ...contentModel.categories
+  ])
+  return {
+    ...contentModel,
+    categories: contentModel.categories.map(attacher),
+    posts: contentModel.posts.map(attacher),
+  }
+}
+
+const _linkPostTags = (post, tags) => {
+  const { permalinkPrefix } = Settings.getSettings()
+  const collectionPath = post.path.split(sep)[0]
+  return {
+    ...post,
+    tags: post.tags.map(tag => {
+      const permalink = makePermalink({
+        prefix: permalinkPrefix,
+        parts: [collectionPath, 'tags', tag]
+      })
+      return {
+        tag,
+        slug: getSlug(tag),
+        permalink
+      }
+    })
+  }
+}
+
+const linkPostTags = (contentModel) => {
+  return {
+    ...contentModel,
+    categories: contentModel.categories.map(category => {
+      return {
+        ...category,
+        posts: category.posts.map(post => _linkPostTags(post, contentModel.tags))
+      }
+    }),
+    posts: contentModel.posts.map(post => _linkPostTags(post, contentModel.tags))
+  }
+}
+
+module.exports = (contentModel) => {
+  return pipeSync(contentModel, [
+    linkPostTags,
+    linkPosts,
+    linkMentionedEntries
+  ])
+}

src/compiler/contentModel2/collection/enhancers/sorting.js

@@ -0,0 +1,16 @@
+const sortPosts = (a, b) => {
+  return new Date(b.publishDate) - new Date(a.publishDate)
+}
+
+module.exports = (contentModel) => {
+  return {
+    ...contentModel,
+    categories: contentModel.categories.map(category => {
+      return {
+        ...category,
+        posts: [...category.posts].sort(sortPosts)
+      }
+    }),
+    posts: [...contentModel.posts].sort(sortPosts)
+  }
+}

src/compiler/contentModel2/collection/enhancers/tags.js

@@ -0,0 +1,38 @@
+const { join } = require('path')
+
+/* Extract tags from posts as an array of tags with posts */
+const getTags = (posts) => {
+  const tags = posts.map((post) => {
+    return post.tags.map((tag) => {
+      return { tag, post }
+    })
+  })
+
+  const flatTags = [].concat(...tags)
+
+  const tagsIndex = flatTags.reduce((acc, tagWithPost) => {
+    const { tag, post } = tagWithPost
+    return {
+      ...acc,
+      [tag.tag]: {
+        ...tag,
+        posts: [
+          ...(acc[tag.tag] ? acc[tag.tag].posts : []),
+          post
+        ]
+      }
+    }
+  }, {})
+
+  return Object
+    .keys(tagsIndex)
+    .map(key => tagsIndex[key])
+    .sort((a, b) => b.posts.length - a.posts.length)
+}
+
+module.exports = (contentModel) => {
+  return {
+    ...contentModel,
+    tags: getTags(contentModel.posts)
+  }
+}