feat: refactor DigestCron to use upsert method for search indexing and remove bulk deletion

Author: omdxp

api/src/digest/cron.ts

@@ -8,7 +8,6 @@ import { GithubService } from "src/github/service";
 import { LoggerService } from "src/logger/service";
 import { ProjectRepository } from "src/project/repository";
 import { RepositoryRepository } from "src/repository/repository";
-import { SearchItem } from "src/search/types";
 import { SearchService } from "src/search/service";
 import { Service } from "typedi";
 
@@ -70,17 +69,13 @@ export class DigestCron {
 
     const projectsFromDataFolder = await this.dataService.listProjects();
 
-    const searchProjectItems: SearchItem[] = [];
-    const searchContributorItems: SearchItem[] = [];
-    const searchContributionItems: SearchItem[] = [];
-
     for (const project of projectsFromDataFolder) {
       const [{ id: projectId }] = await this.projectsRepository.upsert({
         ...project,
         runId,
         id: project.slug,
       });
-      searchProjectItems.push({
+      this.searchService.upsert("project", {
         id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids
         title: project.name,
         type: "project",
@@ -130,7 +125,7 @@ export class DigestCron {
                   runId,
                   id: `${provider}-${githubUser.login}`,
                 });
-              searchContributorItems.push({
+              this.searchService.upsert("contributor", {
                 id: `${provider}-${githubUser.login}`,
                 title: githubUser.name || githubUser.login,
                 type: "contributor",
@@ -158,7 +153,7 @@ export class DigestCron {
                 contributorId,
                 id: `${provider}-${issue.id}`,
               });
-              searchContributionItems.push({
+              this.searchService.upsert("contribution", {
                 id: `${provider}-${issue.id}`,
                 title: issue.title,
                 type: "contribution",
@@ -218,17 +213,6 @@ export class DigestCron {
       await this.contributorsRepository.deleteAllButWithRunId(runId);
       await this.repositoriesRepository.deleteAllButWithRunId(runId);
       await this.projectsRepository.deleteAllButWithRunId(runId);
-      await this.searchService.deleteAllDocuments("project");
-      await this.searchService.deleteAllDocuments("contribution");
-      await this.searchService.deleteAllDocuments("contributor");
-    } catch (error) {
-      captureException(error, { tags: { type: "CRON" } });
-    }
-
-    try {
-      await this.searchService.index("project", searchProjectItems);
-      await this.searchService.index("contribution", searchContributionItems);
-      await this.searchService.index("contributor", searchContributorItems);
     } catch (error) {
       captureException(error, { tags: { type: "CRON" } });
     }

api/src/search/service.ts

@@ -30,25 +30,19 @@ export class SearchService {
     return [];
   };
 
-  public index = async (
+  public upsert = async (
     index: SearchType,
-    data: SearchItem[],
+    data: SearchItem,
   ): Promise<void> => {
     this.logger.info({
-      message: `Indexing ${data.length} items in ${index}`,
+      message: `Upserting ${data.title} in ${index}`,
     });
-    await this.meilisearch.index(index).addDocuments(data);
+    await this.meilisearch.index(index).updateDocuments([data]);
     this.logger.info({
-      message: `Indexed ${data.length} items in ${index}`,
+      message: `Upserted ${data.title} in ${index}`,
     });
   };
 
-  public deleteAllDocuments = async (index: SearchType): Promise<void> => {
-    this.logger.info({ message: `Deleting all documents in ${index}` });
-    await this.meilisearch.index(index).deleteAllDocuments();
-    this.logger.info({ message: `Deleted all documents in ${index}` });
-  };
-
   public ensureIndexes = async (): Promise<void> => {
     await this.meilisearch.createIndex("project");
     this.logger.info({ message: "project index created" });