Added created and modified dates to metadata extractor. (#23)

* Added created and modified dates to metadata extractor.
* Add a sample test for golden files using the new Resource API

Author: ciferkey

src/main/kotlin/com/chimbori/crux/api/Fields.kt

@@ -8,6 +8,8 @@ public object Fields {
   public const val LANGUAGE: String = "language"
   public const val DISPLAY: String = "display"
   public const val ORIENTATION: String = "orientation"
+  public const val PUBLISHED_AT: String = "published_at"
+  public const val MODIFIED_AT: String = "modified_at"
 
   public const val THEME_COLOR_HEX: String = "theme-color-hex"
   public const val THEME_COLOR_HTML: String = "theme-color-html"  // Named colors like "aliceblue"

src/main/kotlin/com/chimbori/crux/extractors/MetadataHelpers.kt

@@ -42,6 +42,16 @@ public fun Document.extractSiteName(): String? = (
 public fun Document.extractThemeColor(): String? =
   select("meta[name=theme-color]").attr("content").nullIfBlank()
 
+public fun Document.extractPublishedAt(): String? = (
+    select("meta[itemprop=dateCreated]").attr("content").nullIfBlank()
+      ?: select("meta[property=article:published_time]").attr("content").nullIfBlank()
+    )?.removeWhiteSpace()?.nullIfBlank()
+
+public fun Document.extractModifiedAt(): String? = (
+    select("meta[itemprop=dateModified]").attr("content").nullIfBlank()
+      ?: select("meta[property=article:modified_time]").attr("content").nullIfBlank()
+    )?.removeWhiteSpace()?.nullIfBlank()
+
 public fun Document.extractKeywords(): List<String> =
   select("meta[name=keywords]").attr("content")
     .removeWhiteSpace()

src/main/kotlin/com/chimbori/crux/plugins/HtmlMetadataExtractor.kt

@@ -4,9 +4,11 @@ import com.chimbori.crux.api.Extractor
 import com.chimbori.crux.api.Fields.AMP_URL
 import com.chimbori.crux.api.Fields.BANNER_IMAGE_URL
 import com.chimbori.crux.api.Fields.CANONICAL_URL
+import com.chimbori.crux.api.Fields.PUBLISHED_AT
 import com.chimbori.crux.api.Fields.DESCRIPTION
 import com.chimbori.crux.api.Fields.FEED_URL
 import com.chimbori.crux.api.Fields.KEYWORDS_CSV
+import com.chimbori.crux.api.Fields.MODIFIED_AT
 import com.chimbori.crux.api.Fields.NEXT_PAGE_URL
 import com.chimbori.crux.api.Fields.PREVIOUS_PAGE_URL
 import com.chimbori.crux.api.Fields.SITE_NAME
@@ -18,10 +20,12 @@ import com.chimbori.crux.common.fetchFromUrl
 import com.chimbori.crux.common.isLikelyArticle
 import com.chimbori.crux.extractors.extractAmpUrl
 import com.chimbori.crux.extractors.extractCanonicalUrl
+import com.chimbori.crux.extractors.extractPublishedAt
 import com.chimbori.crux.extractors.extractDescription
 import com.chimbori.crux.extractors.extractFeedUrl
 import com.chimbori.crux.extractors.extractImageUrl
 import com.chimbori.crux.extractors.extractKeywords
+import com.chimbori.crux.extractors.extractModifiedAt
 import com.chimbori.crux.extractors.extractPaginationUrl
 import com.chimbori.crux.extractors.extractSiteName
 import com.chimbori.crux.extractors.extractThemeColor
@@ -61,6 +65,8 @@ public class HtmlMetadataExtractor(private val okHttpClient: OkHttpClient) : Ext
         DESCRIPTION to resourceToUse.document?.extractDescription(),
         SITE_NAME to resourceToUse.document?.extractSiteName(),
         THEME_COLOR_HEX to resourceToUse.document?.extractThemeColor(),
+        PUBLISHED_AT to resourceToUse.document?.extractPublishedAt(),
+        MODIFIED_AT to resourceToUse.document?.extractModifiedAt(),
         KEYWORDS_CSV to resourceToUse.document?.extractKeywords()?.joinToString(separator = ","),
       ),
       urls = mapOf(

src/test/kotlin/com/chimbori/crux/articles/GoldenFilesTest.kt

@@ -2,6 +2,8 @@ package com.chimbori.crux.articles
 
 import com.chimbori.crux.api.Fields.AMP_URL
 import com.chimbori.crux.api.Fields.BANNER_IMAGE_URL
+import com.chimbori.crux.api.Fields.MODIFIED_AT
+import com.chimbori.crux.api.Fields.PUBLISHED_AT
 import com.chimbori.crux.api.Fields.SITE_NAME
 import com.chimbori.crux.api.Fields.TITLE
 import com.chimbori.crux.common.assertContains
@@ -70,7 +72,27 @@ class GoldenFilesTest {
   }
 
   @Test
-  fun testBBC_AMP() {
+  fun testBBC_AMP_resourceApi() {
+    extractFromFile("http://www.bbc.co.uk/news/amp/37341871".toHttpUrl(), "bbc-amp.html").run {
+      assertEquals("BBC News", fields[SITE_NAME])
+      assertEquals("Baby born on Mediterranean rescue ship", fields[TITLE])
+      assertEquals("http://www.bbc.co.uk/news/amp/37341871".toHttpUrl(), url)
+      assertEquals(
+        "http://ichef.bbci.co.uk/news/999/cpsprodpb/146E6/production/_91168638_baby070012-9-20162-1photocreditalvawhitemsf.jpg".toHttpUrl(),
+        urls[BANNER_IMAGE_URL]
+      )
+      assertStartsWith(
+        "A Nigerian woman has given birth to a boy on board a rescue ship in the Mediterranean after being plucked from an overcrowded rubber dinghy.",
+        article?.text()
+      )
+
+      assertEquals("2016-09-12T14:31:25+00:00", fields[PUBLISHED_AT])
+      assertEquals("2016-09-12T14:31:25+00:00", fields[MODIFIED_AT]);
+    }
+  }
+
+  @Test
+  fun testBBC_AMP_articleApi() {
     fromFile("http://www.bbc.co.uk/news/amp/37341871", "bbc-amp.html").run {
       assertEquals("BBC News", siteName)
       assertEquals("Baby born on Mediterranean rescue ship", title)
@@ -125,7 +147,19 @@ class GoldenFilesTest {
   }
 
   @Test
-  fun testBoingBoing() {
+  fun testBoingBoing_resourceApi() {
+    extractFromFile("http://www.boingboing.net/2010/08/18/dr-laura-criticism-o.html".toHttpUrl(), "boingboing.html").run {
+      assertStartsWith(
+        "Dr. Laura: criticism of me infringes my first amendment rights Dr. Laura Schlessinger is leaving radio to regain her \"first amendment\" rights on the internet.",
+        article?.text()
+      )
+      assertEquals("2010-08-18T01:57:27+00:00", fields[PUBLISHED_AT])
+      assertEquals("2010-08-18T09:43:25+00:00", fields[MODIFIED_AT]);
+    }
+  }
+
+  @Test
+  fun testBoingBoing_articleApi() {
     fromFile("http://www.boingboing.net/2010/08/18/dr-laura-criticism-o.html", "boingboing.html").run {
       assertStartsWith(
         "Dr. Laura: criticism of me infringes my first amendment rights Dr. Laura Schlessinger is leaving radio to regain her \"first amendment\" rights on the internet.",
@@ -1108,7 +1142,24 @@ class GoldenFilesTest {
   }
 
   @Test
-  fun testWallStreetJournal() {
+  fun testWallStreetJournal_resourceApi() {
+    extractFromFile("http://www.wsj.com/articles/SB10001424052748704532204575397061414483040".toHttpUrl(), "wsj.html").run {
+      assertEquals(
+        "https://si.wsj.net/public/resources/images/OB-JO759_0814st_D_20100814143158.jpg".toHttpUrl(),
+        urls[BANNER_IMAGE_URL]
+      )
+      assertStartsWith(
+        "The Obama administration has paid out less than a third of the nearly $230 billion",
+        article?.text()
+      )
+
+      assertEquals("2010-08-14T15:14:00.000Z", fields[PUBLISHED_AT])
+      assertEquals("2010-08-16T04:01:00.000Z", fields[MODIFIED_AT]);
+    }
+  }
+
+  @Test
+  fun testWallStreetJournal_articleApi() {
     fromFile("http://www.wsj.com/articles/SB10001424052748704532204575397061414483040", "wsj.html").run {
       assertStartsWith(
         "The Obama administration has paid out less than a third of the nearly $230 billion",