Instead of using urls, fields and objects, just use a single map metadata that can contain different types of metadata.

Author: chimbori

src/main/kotlin/com/chimbori/crux/api/Fields.kt

@@ -1,6 +1,6 @@
 package com.chimbori.crux.api
 
-/** Well-known keys to use in [Resource.fields] & [Resource.urls]. */
+/** Well-known keys to use in [Resource.metadata]. */
 public object Fields {
   public const val TITLE: String = "title"
   public const val DESCRIPTION: String = "description"

src/main/kotlin/com/chimbori/crux/api/Plugins.kt

@@ -17,10 +17,9 @@ public fun interface Rewriter : Plugin {
  * provided a fully-parsed HTML DOM to extract fields from, and can also make additional HTTP requests if necessary to
  * retrieve additional metadata or to follow redirects.
  *
- * Text fields can be set via the [Resource.fields] property, and URLs via the [Resource.urls] property. Plugins
- * can also rewrite the canonical URL, and can provide an updated DOM tree if the canonical URL is changed. The
- * updated URL and DOM tree will be passed on to the next plugin in sequence, so the exact ordering of plugins is
- * important.
+ * Metadata fields can be set via the [Resource.metadata] property. Plugins can also rewrite the canonical URL, and can
+ * provide an updated DOM tree if the canonical URL is changed. The updated URL and DOM tree will be passed on to the
+ * next plugin in sequence, so the exact ordering of plugins is important.
  */
 public interface Extractor : Plugin {
   /**

src/main/kotlin/com/chimbori/crux/api/Resource.kt

@@ -18,27 +18,11 @@ public data class Resource(
    */
   val article: Element? = null,
 
-  /**
-   * Text fields extracted from this resource, stored as key-value pairs. It is recommended to use well-defined keys
-   * from [com.chimbori.crux.Fields] for all standard fields. Custom fields are also supported, in case none of the
-   * pre-defined keys are applicable.
-   */
-  val fields: Map<String, String?> = emptyMap(),
-
-  /**
-   * URL fields extracted from this resource. Storing these as key-value pairs of [HttpUrl]s avoids re-parsing the same
-   * URLs multiple times. URLs can also be retrieved as strings via the [get] indexed accessor.
-   */
-  val urls: Map<String, HttpUrl?> = emptyMap(),
-
   /** A holder for any kind of custom objects that library users may want to use. */
-  val objects: Map<String, Any?> = emptyMap(),
+  val metadata: Map<String, Any?> = emptyMap(),
 ) {
-  /**
-   * @return value of a named field. If there’s no named [String] field corresponding to this key in [Resource.fields],
-   * but a [HttpUrl] exists in [Resource.urls], the latter will be stringified and returned instead.
-   */
-  public operator fun get(key: String): String? = fields[key] ?: urls[key]?.toString()
+  /** @return value of a named field in [Resource.metadata]. */
+  public operator fun get(key: String): Any? = metadata[key]
 
   /**
    * Merges non-null fields from another [Resource] with this object, and returns a new immutable object. Prefer to use
@@ -48,19 +32,12 @@ public data class Resource(
     url = anotherResource?.url ?: url,
     document = anotherResource?.document ?: document,
     article = anotherResource?.article ?: article,
-    fields = if (anotherResource?.fields == null) fields else fields + anotherResource.fields,
-    urls = if (anotherResource?.urls == null) urls else urls + anotherResource.urls,
-    objects = if (anotherResource?.objects == null) objects else objects + anotherResource.objects,
+    metadata = if (anotherResource?.metadata == null) metadata else metadata + anotherResource.metadata,
   )
 
-  /**
-   * Removes an immutable copy of this [Resource] that only contains non-null values for each key in both [fields]
-   * and [urls].
-   */
+  /** Removes an immutable copy of this [Resource] that only contains non-null values for each key in [metadata]. */
   public fun removeNullValues(): Resource = copy(
-    fields = fields.filterValues { !it.isNullOrBlank() },
-    urls = urls.filterValues { it != null },
-    objects = objects.filterValues { it != null },
+    metadata = metadata.filterValues { it != null },
   )
 
   /** For any potential extension functions to be defined on the companion object. */

src/main/kotlin/com/chimbori/crux/plugins/AmpRedirector.kt

@@ -28,7 +28,7 @@ public class AmpRedirector(
         return if (refetchContentFromCanonicalUrl && canonicalUrl != null) {
           Resource.fetchFromUrl(url = canonicalUrl, okHttpClient = okHttpClient)
         } else {
-          Resource(url = canonicalUrl, urls = mapOf(CANONICAL_URL to canonicalUrl))
+          Resource(url = canonicalUrl, metadata = mapOf(CANONICAL_URL to canonicalUrl))
         }
       }
     }

src/main/kotlin/com/chimbori/crux/plugins/ArticleExtractor.kt

@@ -46,7 +46,7 @@ public class ArticleExtractor(private val okHttpClient: OkHttpClient) : Extracto
 
     val extractedDoc = PostprocessHelpers.postprocess(bestMatchElement)
     return Resource(
-      objects = mapOf(DURATION_MS to extractedDoc.text().estimatedReadingTimeMs()),
+      metadata = mapOf(DURATION_MS to extractedDoc.text().estimatedReadingTimeMs()),
       article = extractedDoc
     )
   }

src/main/kotlin/com/chimbori/crux/plugins/FaviconExtractor.kt

@@ -14,6 +14,6 @@ public class FaviconExtractor : Extractor {
 
   override suspend fun extract(request: Resource): Resource {
     val canonicalUrl = request.document?.extractCanonicalUrl()?.let { request.url?.resolve(it) } ?: request.url
-    return Resource(urls = mapOf(FAVICON_URL to request.document?.extractFaviconUrl(canonicalUrl))).removeNullValues()
+    return Resource(metadata = mapOf(FAVICON_URL to request.document?.extractFaviconUrl(canonicalUrl))).removeNullValues()
   }
 }

src/main/kotlin/com/chimbori/crux/plugins/HtmlMetadataExtractor.kt

@@ -60,17 +60,15 @@ public class HtmlMetadataExtractor(private val okHttpClient: OkHttpClient) : Ext
     return Resource(
       url = if (resourceToUse.url != request.url) resourceToUse.url else null,
       document = resourceToUse.document,
-      fields = mapOf(
+      metadata = mapOf(
+        CANONICAL_URL to canonicalUrl,
         TITLE to resourceToUse.document?.extractTitle(),
         DESCRIPTION to resourceToUse.document?.extractDescription(),
         SITE_NAME to resourceToUse.document?.extractSiteName(),
         THEME_COLOR_HEX to resourceToUse.document?.extractThemeColor(),
         PUBLISHED_AT to resourceToUse.document?.extractPublishedAt(),
         MODIFIED_AT to resourceToUse.document?.extractModifiedAt(),
         KEYWORDS_CSV to resourceToUse.document?.extractKeywords()?.joinToString(separator = ","),
-      ),
-      urls = mapOf(
-        CANONICAL_URL to canonicalUrl,
         NEXT_PAGE_URL to resourceToUse.document?.extractPaginationUrl(resourceToUse.url, "next"),
         PREVIOUS_PAGE_URL to resourceToUse.document?.extractPaginationUrl(resourceToUse.url, "prev"),
         BANNER_IMAGE_URL to resourceToUse.document?.extractImageUrl(canonicalUrl),

src/main/kotlin/com/chimbori/crux/plugins/WebAppManifestParser.kt

@@ -45,17 +45,15 @@ public class WebAppManifestParser(private val okHttpClient: OkHttpClient) : Extr
     val themeColorHtml = manifest.element("theme_color")
     val backgroundColorHtml = manifest.element("background_color")
     return Resource(
-      fields = mapOf(
+      metadata = mapOf(
+        WEB_APP_MANIFEST_URL to webAppManifestUrl,
         TITLE to manifest.element("name"),
         LANGUAGE to manifest.element("lang"),
         DISPLAY to manifest.element("display"),
         ORIENTATION to manifest.element("orientation"),
+        FAVICON_URL to getLargestIconUrl(webAppManifestUrl, manifest?.array<JsonObject>("icons")),
         (if (themeColorHtml?.startsWith("#") == true) THEME_COLOR_HEX else THEME_COLOR_HTML) to themeColorHtml,
         (if (backgroundColorHtml?.startsWith("#") == true) BACKGROUND_COLOR_HEX else BACKGROUND_COLOR_HTML) to backgroundColorHtml,
-      ),
-      urls = mapOf(
-        WEB_APP_MANIFEST_URL to webAppManifestUrl,
-        FAVICON_URL to getLargestIconUrl(webAppManifestUrl, manifest?.array<JsonObject>("icons"))
       )
     ).removeNullValues()
   }

src/test/kotlin/com/chimbori/crux/CruxTest.kt

@@ -111,13 +111,13 @@ class CruxTest {
       override suspend fun extract(request: Resource) =
         Resource(
           url = request.url?.newBuilder()?.encodedPath("/bar")?.build(),
-          fields = mapOf(TITLE to "Foo Title")
+          metadata = mapOf(TITLE to "Foo Title")
         )
     }
 
     val generateTitleForBarPlugin = object : Extractor {
       override fun canExtract(url: HttpUrl) = url.encodedPath == "/bar"
-      override suspend fun extract(request: Resource) = Resource(fields = mapOf(TITLE to "Bar Title"))
+      override suspend fun extract(request: Resource) = Resource(metadata = mapOf(TITLE to "Bar Title"))
     }
 
     // Test Foo before Bar.

src/test/kotlin/com/chimbori/crux/api/ResourceTest.kt

@@ -14,16 +14,15 @@ class ResourceTest {
   fun testResourceMetadataApiExamples() {
     val resource = Resource(
       url = "https://chimbori.com/".toHttpUrl(),
-      fields = mapOf(
+      metadata = mapOf(
         TITLE to "Life, the Universe, and Everything",
-        DESCRIPTION to "42"
-      ), urls = mapOf(
+        DESCRIPTION to "42",
         CANONICAL_URL to "https://chimbori.com/".toHttpUrl()
       )
     )
     assertEquals("Life, the Universe, and Everything", resource[TITLE])
     assertEquals("42", resource[DESCRIPTION])
-    assertEquals("https://chimbori.com/".toHttpUrl(), resource.urls[CANONICAL_URL])
-    assertNull(resource.urls[BANNER_IMAGE_URL])
+    assertEquals("https://chimbori.com/".toHttpUrl(), resource[CANONICAL_URL])
+    assertNull(resource[BANNER_IMAGE_URL])
   }
 }