Refactor: Convert to a group of utility functions

README.md

@@ -21,48 +21,30 @@ npm install bsky-tldr
 
 Exports from the library:
 
-`getDailyPostsFromFollows` is the main function to retrieve daily posts from follows. See Data Structure Example below.
+Utility functions that wrap the AT Protocol pagination with JavaScript generators:
 
-There are also utility functions that wrap the AtProto pagination with JavaScript generators:
-
-- `retrieveAuthorFeedGenerator()` is a generator function to retrieve posts from an author and
-- `retrieveFollowsGenerator` is a generator function to retrieve follows from an author.
+- `retrieveFollows()` is a generator function to retrieve follows from an author.
+- `retrieveAuthorFeed()` is a generator function to retrieve posts from an author, for a specific date.
 
 And if you want to convert post `uri` to a public URL:
 
 - `uriToUrl` is a utility function to convert a post uri to a public url to view post on the web.
 
 ## Data Structure Example
 
-Here's the data structure built with our `getDailyPostsFromFollows` library function for viewing posts from your follows. If you're only following 1 user, and they had two posts on January 31, 2025:
+Here's the post data structure returned from our `retrieveAuthorFeed` function for viewing posts for a specific author:
 
 ```json
 {
-  "follows": {
-    "did:plc:oeio7zuhrsvmlyhia7e44nk6": {
-      "handle": "mattpocock.com",
-      "posts": [
-        {
-          "uri": "at://did:plc:oeio7zuhrsvmlyhia7e44nk6/app.bsky.feed.post/3lgzvm46vhu2c",
-          "content": "TIL about process.exitCode = 1;\n\nUseful if you want to mark a process as failed without immediately exiting it",
-          "createdAt": "2025-01-31T11:32:00.769Z",
-          "isRepost": false,
-          "links": []
-        },
-        {
-          "uri": "at://did:plc:oeio7zuhrsvmlyhia7e44nk6/app.bsky.feed.post/3lh2c4nddwr2s",
-          "content": "Is there a decent chunking algorithm library on NPM?\n\nI know Langchain and LlamaIndex have some, but figured there were probably some unbundled from frameworks.\n\nChunking: chunking text documents to be fed into a RAG system.",
-          "createdAt": "2025-01-31T15:16:00.525Z",
-          "isRepost": false,
-          "links": []
-        }
-      ]
-    }
-  }
+  "uri": "at://did:plc:oeio7zuhrsvmlyhia7e44nk6/app.bsky.feed.post/3lgzvm46vhu2c",
+  "content": "TIL about process.exitCode = 1;\n\nUseful if you want to mark a process as failed without immediately exiting it",
+  "createdAt": "2025-01-31T11:32:00.769Z",
+  "isRepost": false,
+  "links": []
 }
 ```
 
-The author's `did` and `handle` are provided, along with posts that include `uri`, `content`, `createdAt`, `isRepost` (`false` means it's an original by the author) and `links` which are the full links mentioned in the post.
+Posts that include `uri`, `content`, `createdAt`, `isRepost` (`false` means it's an original by the author) and `links` which are the full links mentioned in the post.
 
 If you need more information in your app, use `@atproto/api` library directly to retrieve the author's profile using their `did`, or the full post and replies via its `uri`.
 
@@ -77,14 +59,7 @@ BLUESKY_PASSWORD=
 
 2. Update script:
 
-First change `sourceActor` and `targetDate` in `./src/scripts/retrieve-posts.ts` to your Bluesky handle or `did`, and a date in `yyyymmdd` format.
-
-```javascript
-const postsPerAuthorResponse = await buildDailyPostsFromFollows({
-  sourceActor: 'brianfive.xyz', // or 'did:plc:3cgdoyodzdnhugjjrazljkzq'
-  targetDate: '20250201',
-});
-```
+Change `SOURCE_ACTOR`, `TARGET_DATE` and `TIMEZONE_OFFSET` in `./src/scripts/retrieve-posts.ts` to your Bluesky handle or `did`, and a date in `yyyymmdd` format.
 
 3. Run the script:
 

package.json

@@ -30,7 +30,8 @@
   "scripts": {
     "build": "rollup -c",
     "dev": "rollup -c --watch",
-    "test": "vitest",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "coverage": "vitest run --coverage",
     "prepare": "npm run build",
     "retrievePosts": "tsx src/scripts/retrieve-posts.ts"

src/index.ts

@@ -1,32 +1,5 @@
-// Import types
 import type { Follow, Post } from './lib/bsky-tldr';
-import type {
-  AuthorFeed,
-  DailyPostsFromFollows,
-  DailyPostsFromFollowsResponse,
-} from './lib/getDailyPostsFromFollows';
-
-// Import functions and classes
-import {
-  retrieveAuthorFeedGenerator,
-  retrieveFollowsGenerator,
-} from './lib/bsky-tldr';
-import { getDailyPostsFromFollows } from './lib/getDailyPostsFromFollows';
+import { retrieveAuthorFeed, retrieveFollows } from './lib/bsky-tldr';
 import { uriToUrl } from './lib/uriToUrl';
-
-// Types
-export type {
-  AuthorFeed,
-  DailyPostsFromFollows,
-  DailyPostsFromFollowsResponse,
-  Follow,
-  Post,
-};
-
-// Core functionality
-export {
-  getDailyPostsFromFollows,
-  retrieveAuthorFeedGenerator,
-  retrieveFollowsGenerator,
-  uriToUrl,
-};
+export { retrieveAuthorFeed, retrieveFollows, uriToUrl };
+export type { Follow, Post };

src/lib/bsky-tldr.ts

@@ -7,6 +7,7 @@ import {
   isLink,
   Main,
 } from '@atproto/api/dist/client/types/app/bsky/richtext/facet';
+import { earlierThenTargetDate, withinTargetDate } from './target-date';
 
 export interface Follow {
   did: string;
@@ -21,7 +22,7 @@ export interface Post {
   links: string[];
 }
 
-export async function* retrieveFollowsGenerator({
+export async function* retrieveFollows({
   bluesky,
   actor,
   batchSize = 50,
@@ -66,14 +67,18 @@ export async function* retrieveFollowsGenerator({
   }
 }
 
-export async function* retrieveAuthorFeedGenerator({
+export async function* retrieveAuthorFeed({
   bluesky,
   actor,
   batchSize = 5,
+  targetDate,
+  timezoneOffset = 0,
 }: {
   bluesky: Agent;
   actor: string;
   batchSize?: number;
+  targetDate: string;
+  timezoneOffset?: number;
 }): AsyncGenerator<Post, void, undefined> {
   if (!bluesky) {
     throw new Error('Bluesky client not initialized');
@@ -91,24 +96,36 @@ export async function* retrieveAuthorFeedGenerator({
       });
 
       for (const feedViewPost of data.feed) {
-        if (!validateFeedViewPost(feedViewPost)) {
-          console.info('Invalid feed view post:', feedViewPost);
+        const v = validateFeedViewPost(feedViewPost);
+        if (!v.success) {
+          // console.info('Invalid feed view post, skipping due to', v.error);
           continue;
         }
 
         const postView = feedViewPost.post;
         const record = postView.record;
-        const facets = record.facets as Main[];
-        const links = extractLinks(facets);
 
-        yield {
-          uri: postView.uri,
-          content: (record.text as string) || '',
-          createdAt: (record.createdAt as string) || '',
-          isRepost: isReasonRepost(feedViewPost.reason),
-          links,
-        };
-        count++;
+        const postTime = new Date(record.createdAt as string);
+
+        // If post is from before target date, we can stop processing posts for this author
+        if (earlierThenTargetDate(postTime, targetDate, timezoneOffset)) {
+          return;
+        }
+
+        // Only include posts from target date (between start and end of day)
+        if (withinTargetDate(postTime, targetDate, timezoneOffset)) {
+          const facets = record.facets as Main[];
+          const links = extractLinks(facets);
+
+          yield {
+            uri: postView.uri,
+            content: (record.text as string) || '',
+            createdAt: (record.createdAt as string) || '',
+            isRepost: isReasonRepost(feedViewPost.reason),
+            links,
+          };
+          count++;
+        }
       }
 
       cursor = data.cursor;

src/lib/target-date.ts

@@ -38,6 +38,9 @@ export function withinTargetDate(
  * @returns
  */
 export function targetDateRange(yyyymmdd: string, timezoneOffset: number = 0) {
+  if (yyyymmdd.length !== 8) {
+    throw new Error('yyyymmdd must be 8 characters');
+  }
   const year = yyyymmdd.slice(0, 4);
   const month = yyyymmdd.slice(4, 6);
   const day = yyyymmdd.slice(6, 8);
@@ -66,3 +69,16 @@ export function targetDateRange(yyyymmdd: string, timezoneOffset: number = 0) {
 function isValidDate(date: Date) {
   return date instanceof Date && !isNaN(date.getTime());
 }
+
+/**
+ *
+ * @returns yesterday's date in YYYYMMDD format
+ */
+export function getYesterday() {
+  const today = new Date();
+  const yesterday = new Date(today.getTime() - 86400000);
+  const year = yesterday.getFullYear();
+  const month = String(yesterday.getMonth() + 1).padStart(2, '0');
+  const day = String(yesterday.getDate()).padStart(2, '0');
+  return `${year}${month}${day}`;
+}