Parse RSS1/RSS2/ATOM/JSONFeed tags/categories

Shinokuni · Shinokuni · commit 455194ec728c · 2025-07-15T16:23:04.000+02:00
diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemAdapter.kt
@@ -10,13 +10,15 @@ import com.readrops.api.utils.extensions.nonNullText
 import com.readrops.api.utils.extensions.nullableText
 import com.readrops.api.utils.extensions.nullableTextRecursively
 import com.readrops.db.entities.Item
+import com.readrops.db.entities.Tag
 import com.readrops.db.util.DateUtils
 import java.time.LocalDateTime
 
 class ATOMItemAdapter : XmlAdapter<Item> {
 
     override fun fromXml(konsumer: Konsumer): Item {
         val item = Item()
+        val tags = arrayListOf<Tag>()
 
         return item.apply {
             konsumer.allChildrenAutoIgnore(names) {
@@ -36,10 +38,18 @@ class ATOMItemAdapter : XmlAdapter<Item> {
                     "summary" -> description = nullableTextRecursively()
                     "content" -> content = nullableTextRecursively()
                     "media:group" -> RSSMedia.parseMediaGroup(this, item)
+                    "category" -> {
+                        attributes.getValueOrNull("term")?.let {
+                            tags += Tag(name = it)
+                        }
+                    }
+
                     else -> skipContents()
                 }
             }
 
+            this.tags = tags
+
             validateItem(this)
             if (pubDate == null) pubDate = LocalDateTime.now()
             if (remoteId == null) remoteId = link
@@ -61,7 +71,7 @@ class ATOMItemAdapter : XmlAdapter<Item> {
     companion object {
         val names = Names.of(
             "title", "id", "updated", "link", "author", "summary",
-            "content", "group", "published"
+            "content", "group", "published", "category"
         )
     }
 }
diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt
@@ -5,6 +5,7 @@ import com.readrops.api.utils.exceptions.ParseException
 import com.readrops.api.utils.extensions.nextNonEmptyString
 import com.readrops.api.utils.extensions.nextNullableString
 import com.readrops.db.entities.Item
+import com.readrops.db.entities.Tag
 import com.readrops.db.util.DateUtils
 import com.squareup.moshi.JsonAdapter
 import com.squareup.moshi.JsonReader
@@ -42,6 +43,7 @@ class JSONItemsAdapter : JsonAdapter<List<Item>>() {
                         7 -> pubDate = DateUtils.parse(nextNullableString())
                         8 -> author = parseAuthor(reader) // jsonfeed 1.0
                         9 -> author = parseAuthors(reader) // jsonfeed 1.1
+                        10 -> tags = parseTags(reader)
                         else -> skipValue()
                     }
                 }
@@ -88,14 +90,41 @@ class JSONItemsAdapter : JsonAdapter<List<Item>>() {
             authors.filterNotNull().joinToString(limit = AUTHORS_MAX) else null
     }
 
+    private fun parseTags(reader: JsonReader): List<Tag> = with(reader) {
+        val tags = arrayListOf<Tag>()
+        beginArray()
+
+        while (hasNext()) {
+            val name = nextNullableString()
+
+            if (!name.isNullOrEmpty()) {
+                tags += Tag(name = name)
+            }
+        }
+
+        endArray()
+        tags
+    }
+
     private fun validateItem(item: Item): Boolean = when {
         item.title == null -> throw ParseException("Item title is required")
         item.link == null -> throw ParseException("Item link is required")
         else -> true
     }
 
     companion object {
-        val names: JsonReader.Options = JsonReader.Options.of("id", "url", "title",
-                "content_html", "content_text", "summary", "image", "date_published", "author", "authors")
+        val names: JsonReader.Options = JsonReader.Options.of(
+            "id",
+            "url",
+            "title",
+            "content_html",
+            "content_text",
+            "summary",
+            "image",
+            "date_published",
+            "author",
+            "authors",
+            "tags"
+        )
     }
 }
diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemAdapter.kt
@@ -10,13 +10,15 @@ import com.readrops.api.utils.extensions.nonNullText
 import com.readrops.api.utils.extensions.nullableText
 import com.readrops.api.utils.extensions.nullableTextRecursively
 import com.readrops.db.entities.Item
+import com.readrops.db.entities.Tag
 import com.readrops.db.util.DateUtils
 import java.time.LocalDateTime
 
 class RSS1ItemAdapter : XmlAdapter<Item> {
 
     override fun fromXml(konsumer: Konsumer): Item {
         val item = Item()
+        val tags = arrayListOf<Tag>()
 
         val authors = arrayListOf<String?>()
         val about = konsumer.attributes.getValueOrNull(
@@ -33,10 +35,17 @@ class RSS1ItemAdapter : XmlAdapter<Item> {
                     "dc:creator" -> authors += nullableText()
                     "description" -> description = nullableTextRecursively()
                     "content:encoded" -> content = nullableTextRecursively()
+                    "dc:subject" -> {
+                        nullableText()?.let {
+                            tags += Tag(name = it)
+                        }
+                    }
+
                     else -> skipContents()
                 }
             }
 
+            this.tags = tags
             validateItem(this)
 
             if (pubDate == null) pubDate = LocalDateTime.now()
@@ -54,6 +63,7 @@ class RSS1ItemAdapter : XmlAdapter<Item> {
     }
 
     companion object {
-        val names = Names.of("title", "description", "date", "link", "creator", "encoded")
+        val names =
+            Names.of("title", "description", "date", "link", "creator", "encoded", "subject")
     }
 }
diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemAdapter.kt
@@ -12,6 +12,7 @@ import com.readrops.api.utils.extensions.nonNullText
 import com.readrops.api.utils.extensions.nullableText
 import com.readrops.api.utils.extensions.nullableTextRecursively
 import com.readrops.db.entities.Item
+import com.readrops.db.entities.Tag
 import com.readrops.db.util.DateUtils
 import java.time.LocalDateTime
 
@@ -21,6 +22,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {
         val item = Item()
 
         val creators = arrayListOf<String?>()
+        val tags = arrayListOf<Tag>()
 
         return item.apply {
             konsumer.allChildrenAutoIgnore(names) {
@@ -37,9 +39,17 @@ class RSS2ItemAdapter : XmlAdapter<Item> {
                     "enclosure" -> RSSMedia.parseMediaContent(this, item = this@apply)
                     "media:content" -> RSSMedia.parseMediaContent(this, item = this@apply)
                     "media:group" -> RSSMedia.parseMediaGroup(this, item = this@apply)
+                    "category" -> {
+                        nullableText()?.let {
+                            tags += Tag(name = it)
+                        }
+                    }
+
                     else -> skipContents() // for example media:description
                 }
             }
+
+            this.tags = tags
             finalizeItem(this, creators)
         }
     }
@@ -63,7 +73,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {
     companion object {
         val names = Names.of(
             "title", "link", "author", "creator", "pubDate", "date",
-            "guid", "description", "encoded", "enclosure", "content", "group"
+            "guid", "description", "encoded", "enclosure", "content", "group", "category"
         )
     }
 }
diff --git a/api/src/test/java/com/readrops/api/localfeed/atom/ATOMAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/atom/ATOMAdapterTest.kt
@@ -38,6 +38,7 @@ class ATOMAdapterTest {
             assertEquals(description, "Summary")
             assertEquals(remoteId, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac")
             TestCase.assertNotNull(content)
+            assertEquals(3, tags.size)
         }
     }
 
diff --git a/api/src/test/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt
@@ -47,6 +47,8 @@ class JSONFeedAdapterTest {
             assertEquals(pubDate, DateUtils.parse("2017-09-25T14:27:27-07:00"))
             assertEquals(author, "Author 1")
             TestCase.assertNotNull(content)
+
+            assertEquals(3, tags.size)
         }
 
     }
diff --git a/api/src/test/java/com/readrops/api/localfeed/rss1/RSS1AdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/rss1/RSS1AdapterTest.kt
@@ -42,6 +42,7 @@ class RSS1AdapterTest {
             assertEquals(author, "msmash")
             assertNotNull(description)
             assertEquals(content, "content:encoded")
+            assertEquals(1, tags.size)
         }
     }
 
diff --git a/api/src/test/java/com/readrops/api/localfeed/rss2/RSS2AdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/rss2/RSS2AdapterTest.kt
@@ -38,6 +38,7 @@ class RSS2AdapterTest {
             assertEquals(author, "Author 1")
             assertEquals(description, "<a href=\"https://news.ycombinator.com/item?id=24273602\">Comments</a>")
             assertEquals(remoteId, "https://www.bbc.com/news/world-africa-53887947")
+            assertEquals(3, tags.size)
         }
     }
 
diff --git a/api/src/test/resources/localfeed/atom/atom_items.xml b/api/src/test/resources/localfeed/atom/atom_items.xml
@@ -22,6 +22,9 @@
         <content type="html">
             &lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Add an option to open item url in custom tab&lt;/pre&gt;
         </content>
+        <category term="category 1"/>
+        <category term="category 2"/>
+        <category term="category 3"/>
     </entry>
     <entry>
         <id>tag:github.com,2008:Grit::Commit/e0945823eecf269e5beea646ac5d7e630e08afbf</id>
diff --git a/api/src/test/resources/localfeed/json/json_feed.json b/api/src/test/resources/localfeed/json/json_feed.json
@@ -18,7 +18,12 @@
       "author": {
         "url": "this is an url",
         "name": "Author 1"
-      }
+      },
+      "tags": [
+        "Tag 1",
+        "Tag 2",
+        "Tag 3"
+      ]
     },
     {
       "id": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html",
diff --git a/api/src/test/resources/localfeed/rss2/rss_feed.xml b/api/src/test/resources/localfeed/rss2/rss_feed.xml
@@ -19,6 +19,9 @@
          <author>Author 1</author>
          <description><![CDATA[<a href="https://news.ycombinator.com/item?id=24273602">Comments</a>]]></description>
          <media:description>media description</media:description>
+         <category>category 1</category>
+         <category>category 2</category>
+         <category>category 3</category>
       </item>
       <item>
          <title>Palantir S-1</title>
diff --git a/app/src/main/java/com/readrops/app/repositories/GReaderRepository.kt b/app/src/main/java/com/readrops/app/repositories/GReaderRepository.kt
@@ -190,7 +190,7 @@ class GReaderRepository(
 
     private suspend fun insertItemsTags(newItems: List<Item>, allTags: List<Tag>) {
         newItems.associate { it to it.tags }
-            .map { (item, tags) ->
+            .flatMap { (item, tags) ->
                 tags
                     .filter { tag -> allTags.any { tag.remoteId == it.remoteId } }
                     .map { tag ->
@@ -200,7 +200,6 @@ class GReaderRepository(
                         )
                     }
             }
-            .flatten()
             .run {
                 database.tagJoinDao().insert(this)
             }
diff --git a/app/src/main/java/com/readrops/app/repositories/LocalRSSRepository.kt b/app/src/main/java/com/readrops/app/repositories/LocalRSSRepository.kt
@@ -9,6 +9,8 @@ import com.readrops.app.util.Utils
 import com.readrops.db.Database
 import com.readrops.db.entities.Feed
 import com.readrops.db.entities.Item
+import com.readrops.db.entities.Tag
+import com.readrops.db.entities.TagJoin
 import com.readrops.db.entities.account.Account
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.Dispatchers
@@ -30,7 +32,8 @@ class LocalRSSRepository(
     private val dispatcher: CoroutineDispatcher = Dispatchers.IO
 ) : BaseRepository(database, account), KoinComponent {
 
-    override suspend fun login(account: Account) { /* useless here */ }
+    override suspend fun login(account: Account) { /* useless here */
+    }
 
     override suspend fun synchronize(
         selectedFeeds: List<Feed>,
@@ -60,7 +63,19 @@ class LocalRSSRepository(
                         try {
                             val pair = dataSource.queryRSSResource(feed.url!!, headers.build())
 
-                            pair?.let { newItems.addAll(insertNewItems(it.second, feed)) }
+                            pair?.let { (_, pairItems) ->
+                                val itemsTags = pairItems.flatMap { it.tags }
+                                    .distinct()
+                                val allTags = insertTags(itemsTags)
+
+                                val pairNewItems = insertNewItems(pairItems, feed)
+
+                                if (itemsTags.isNotEmpty()) {
+                                    insertItemsTags(pairNewItems, allTags)
+                                }
+
+                                newItems.addAll(pairNewItems)
+                            }
                         } catch (e: Exception) {
                             errors[feed] = e
                         }
@@ -106,6 +121,26 @@ class LocalRSSRepository(
         errors
     }
 
+    private suspend fun insertTags(tags: List<Tag>): List<Tag> {
+        database.tagDao().insertConflict(tags.map { it.copy(accountId = account.id) })
+        return database.tagDao().selectAll(account.id)
+    }
+
+    private suspend fun insertItemsTags(newItems: List<Item>, allTags: List<Tag>) {
+        newItems.associate { it to it.tags }
+            .flatMap { (item, tags) ->
+                tags.map { tag ->
+                    TagJoin(
+                        itemId = item.id,
+                        tagId = allTags.first { tag.name == it.name }.id,
+                    )
+                }
+            }
+            .run {
+                database.tagJoinDao().insertConflict(this)
+            }
+    }
+
     private suspend fun insertNewItems(items: List<Item>, feed: Feed): List<Item> {
         val newItems = mutableListOf<Item>()
 
diff --git a/db/schemas/com.readrops.db.Database/6.json b/db/schemas/com.readrops.db.Database/6.json
@@ -2,7 +2,7 @@
   "formatVersion": 1,
   "database": {
     "version": 6,
-    "identityHash": "a7510157ca821655f8b155517db6c780",
+    "identityHash": "b359d0af3b75281c9d9753b1c2011a2c",
     "entities": [
       {
         "tableName": "Feed",
@@ -390,7 +390,7 @@
       },
       {
         "tableName": "Tag",
-        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `remote_id` TEXT NOT NULL, `account_id` INTEGER NOT NULL, FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `remote_id` TEXT, `account_id` INTEGER NOT NULL, FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
         "fields": [
           {
             "fieldPath": "id",
@@ -407,8 +407,7 @@
           {
             "fieldPath": "remoteId",
             "columnName": "remote_id",
-            "affinity": "TEXT",
-            "notNull": true
+            "affinity": "TEXT"
           },
           {
             "fieldPath": "accountId",
@@ -651,7 +650,7 @@
     ],
     "setupQueries": [
       "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
-      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a7510157ca821655f8b155517db6c780')"
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'b359d0af3b75281c9d9753b1c2011a2c')"
     ]
   }
 }
diff --git a/db/src/main/java/com/readrops/db/dao/TagDao.kt b/db/src/main/java/com/readrops/db/dao/TagDao.kt
diff --git a/db/src/main/java/com/readrops/db/dao/TagJoinDao.kt b/db/src/main/java/com/readrops/db/dao/TagJoinDao.kt
diff --git a/db/src/main/java/com/readrops/db/entities/Tag.kt b/db/src/main/java/com/readrops/db/entities/Tag.kt

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ import com.readrops.api.utils.extensions.nonNullText`
`12`	`12`	`import com.readrops.api.utils.extensions.nullableText`
`13`	`13`	`import com.readrops.api.utils.extensions.nullableTextRecursively`
`14`	`14`	`import com.readrops.db.entities.Item`
	`15`	`+import com.readrops.db.entities.Tag`
`15`	`16`	`import com.readrops.db.util.DateUtils`
`16`	`17`	`import java.time.LocalDateTime`
`17`	`18`
`@@ -21,6 +22,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {`
`21`	`22`	`val item = Item()`
`22`	`23`
`23`	`24`	`val creators = arrayListOf<String?>()`
	`25`	`+ val tags = arrayListOf<Tag>()`
`24`	`26`
`25`	`27`	`return item.apply {`
`26`	`28`	`konsumer.allChildrenAutoIgnore(names) {`
`@@ -37,9 +39,17 @@ class RSS2ItemAdapter : XmlAdapter<Item> {`
`37`	`39`	`"enclosure" -> RSSMedia.parseMediaContent(this, item = this@apply)`
`38`	`40`	`"media:content" -> RSSMedia.parseMediaContent(this, item = this@apply)`
`39`	`41`	`"media:group" -> RSSMedia.parseMediaGroup(this, item = this@apply)`
	`42`	`+ "category" -> {`
	`43`	`+ nullableText()?.let {`
	`44`	`+ tags += Tag(name = it)`
	`45`	`+ }`
	`46`	`+ }`
	`47`	`+`
`40`	`48`	`else -> skipContents() // for example media:description`
`41`	`49`	`}`
`42`	`50`	`}`
	`51`	`+`
	`52`	`+ this.tags = tags`
`43`	`53`	`finalizeItem(this, creators)`
`44`	`54`	`}`
`45`	`55`	`}`
`@@ -63,7 +73,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {`
`63`	`73`	`companion object {`
`64`	`74`	`val names = Names.of(`
`65`	`75`	`"title", "link", "author", "creator", "pubDate", "date",`
`66`		`- "guid", "description", "encoded", "enclosure", "content", "group"`
	`76`	`+ "guid", "description", "encoded", "enclosure", "content", "group", "category"`
`67`	`77`	`)`
`68`	`78`	`}`
`69`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ class ATOMAdapterTest {`
`38`	`38`	`assertEquals(description, "Summary")`
`39`	`39`	`assertEquals(remoteId, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac")`
`40`	`40`	`TestCase.assertNotNull(content)`
	`41`	`+ assertEquals(3, tags.size)`
`41`	`42`	`}`
`42`	`43`	`}`
`43`	`44`
Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,8 @@ class JSONFeedAdapterTest {`
`47`	`47`	`assertEquals(pubDate, DateUtils.parse("2017-09-25T14:27:27-07:00"))`
`48`	`48`	`assertEquals(author, "Author 1")`
`49`	`49`	`TestCase.assertNotNull(content)`
	`50`	`+`
	`51`	`+ assertEquals(3, tags.size)`
`50`	`52`	`}`
`51`	`53`
`52`	`54`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@ class RSS1AdapterTest {`
`42`	`42`	`assertEquals(author, "msmash")`
`43`	`43`	`assertNotNull(description)`
`44`	`44`	`assertEquals(content, "content:encoded")`
	`45`	`+ assertEquals(1, tags.size)`
`45`	`46`	`}`
`46`	`47`	`}`
`47`	`48`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ class RSS2AdapterTest {`
`38`	`38`	`assertEquals(author, "Author 1")`
`39`	`39`	`assertEquals(description, "<a href=\"https://news.ycombinator.com/item?id=24273602\">Comments</a>")`
`40`	`40`	`assertEquals(remoteId, "https://www.bbc.com/news/world-africa-53887947")`
	`41`	`+ assertEquals(3, tags.size)`
`41`	`42`	`}`
`42`	`43`	`}`
`43`	`44`