Tweak to get number of articles for each word, not just occurrences

Author: sveinbjornt

db/sql.py

@@ -464,3 +464,59 @@ def frequency(
                 datefmt=datefmt,
             )
         return result
+
+
+class ArticleFrequencyQuery(_BaseQuery):
+    """A query yielding the number of articles a given word occurs in
+    over a given period of time, broken down by either
+    day or week."""
+
+    _Q = """
+        with days as (
+            select to_char(d, :datefmt) date
+            from generate_series(
+                :start,
+                :end,
+                :timeunit
+            ) d
+        ),
+        appearances as (
+            select to_char(a.timestamp, :datefmt) date, count(distinct w.article_id) cnt
+            from words w, articles a
+            where w.stem = :stem
+            and w.cat = :cat
+            and w.article_id = a.id
+            and a.timestamp >= :start
+            and a.timestamp <= :end
+            group by date
+            order by date
+        )
+        select days.date, coalesce(appearances.cnt,0) from days
+        left outer join appearances on days.date = appearances.date;
+        """
+
+    @classmethod
+    def frequency(
+        cls,
+        stem: str,
+        cat: str,
+        start: datetime,
+        end: datetime,
+        timeunit: str = "day",
+        enclosing_session: Optional[Session] = None,
+    ) -> Iterable[Any]:
+        result: Iterable[Any] = []
+        with SessionContext(session=enclosing_session, read_only=True) as session:
+            assert timeunit in ["week", "day"]
+            datefmt = "IYYY-IW" if timeunit == "week" else "YYYY-MM-DD"
+            tu = f"1 {timeunit}"
+            result = cls().execute(
+                session,
+                stem=stem,
+                cat=cat,
+                start=start,
+                end=end,
+                timeunit=tu,
+                datefmt=datefmt,
+            )
+        return result

get_word_frequencies.py

@@ -1,6 +1,6 @@
 import csv
 from datetime import datetime, timedelta, date, time
-from db.sql import WordFrequencyQuery
+from db.sql import WordFrequencyQuery, ArticleFrequencyQuery
 from settings import Settings
 
 def get_frequencies_to_csv_per_word():
@@ -14,10 +14,10 @@ def get_frequencies_to_csv_per_word():
     # Set the end date to the end of yesterday to exclude the current, unfinished day.
     today = date.today()
     end_date = datetime.combine(today, time.min) - timedelta(seconds=1)
-    start_date = end_date - timedelta(days=7 * 365)  # 7 years back
+    start_date = end_date - timedelta(days=(7 * 365) - 1)
 
     words_to_check = [
-        # ("stýrivextir", "kk"),
+        ("stýrivextir", "kk"),
         ("verðbólga", "kvk")
     ]
 
@@ -27,19 +27,36 @@ def get_frequencies_to_csv_per_word():
 
         with open(output_filename, 'w', newline='', encoding='utf-8') as csvfile:
             csv_writer = csv.writer(csvfile)
-            csv_writer.writerow(['date', 'count'])
+            csv_writer.writerow(['date', 'count', 'article_count'])
 
-            results = WordFrequencyQuery.frequency(
+            word_freq = WordFrequencyQuery.frequency(
                 stem=stem,
                 cat=cat,
                 start=start_date,
                 end=end_date,
                 timeunit="day"
             )
 
+            article_freq = ArticleFrequencyQuery.frequency(
+                stem=stem,
+                cat=cat,
+                start=start_date,
+                end=end_date,
+                timeunit="day"
+            )
+
+            # Create dictionaries for quick lookup
+            word_freq_dict = {d: c for d, c in word_freq}
+            article_freq_dict = {d: c for d, c in article_freq}
+
+            # Get all unique dates from both queries
+            all_dates = sorted(list(set(word_freq_dict.keys()) | set(article_freq_dict.keys())))
+
             rows_written = 0
-            for d, count in results:
-                csv_writer.writerow([d, count])
+            for d in all_dates:
+                count = word_freq_dict.get(d, 0)
+                article_count = article_freq_dict.get(d, 0)
+                csv_writer.writerow([d, count, article_count])
                 rows_written += 1
 
             if rows_written > 0: