Fix 500 server error on project dashboard (#1779)

Large projects were returning an HTTP 500 on the project dashboard,
because PHP was running out of memory trying to return every entry for
stats to be calculated client-side. This changes the stats to be
calculated server-side instead. We get rid of the "number of entries
with audio" stat because it's not possible to calculate in a simple
Mongo query and would require serializing the entire entry database
server-side, causing the very memory error we're trying to get rid of.

Author: rmunn

next-app/src/routes/projects/[project_code]/+page.svelte

@@ -33,12 +33,6 @@
 			icon: NotesIcon,
 			url: `/app/lexicon/${ project.id }`,
 		},
-		{
-			title: 'Entries with audio',
-			value: project.num_entries_with_audio,
-			icon: VoiceIcon,
-			url: `/app/lexicon/${ project.id }#!/editor/entry/000000?filterBy=Audio`,
-		},
 		{
 			title: 'Entries with pictures',
 			value: project.num_entries_with_pictures,

next-app/src/routes/projects/[project_code]/meta/+server.ts

@@ -9,8 +9,9 @@ type LegacyProjectDetails = {
 }
 
 type LegacyStats = {
-	entries: object[],
-	comments: Comment[],
+	num_entries,
+	num_entries_with_pictures,
+	num_unresolved_comments,
 }
 
 type Comment = {
@@ -23,40 +24,27 @@ export type ProjectDetails = {
 	name: string,
 	num_users: number,
 	num_entries: number,
-	num_entries_with_audio: number,
 	num_entries_with_pictures: number,
 	num_unresolved_comments?: number,
 }
 
 export async function fetch_project_details({ project_code, cookie }) {
 	const { id, projectName: name, users }: LegacyProjectDetails = await sf({ name: 'set_project', args: [ project_code ], cookie })
-	const { entries, comments }: LegacyStats = await sf({ name: 'lex_stats', cookie })
+	const stats: LegacyStats = await sf({ name: 'lex_stats', cookie })
 
 	const details: ProjectDetails = {
 		id,
 		code: project_code,
 		name,
 		num_users: Object.keys(users).length,
-		num_entries: entries.length,
-		num_entries_with_audio: entries.filter(has_audio).length,
-		num_entries_with_pictures: entries.filter(has_picture).length,
+		num_entries: stats.num_entries,
+		num_entries_with_pictures: stats.num_entries_with_pictures,
 	}
 
 	const { role } = await fetch_current_user(cookie)
 	if (can_view_comments(role)) {
-		const unresolved_comments = comments.filter(({ status }) => status !== 'resolved')
-
-		details.num_unresolved_comments = unresolved_comments.length
+		details.num_unresolved_comments = stats.num_unresolved_comments
 	}
 
 	return details
 }
-
-function has_picture(entry: object) {
-	return JSON.stringify(entry).includes('"pictures":')
-}
-
-// audio can be found in lots of places other than lexeme, ref impl used: https://github.com/sillsdev/web-languageforge/blob/develop/src/angular-app/bellows/core/offline/editor-data.service.ts#L523
-function has_audio(entry: object) {
-	return JSON.stringify(entry).includes('-audio":') // naming convention imposed by src/angular-app/languageforge/lexicon/settings/configuration/input-system-view.model.ts L81
-}

src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace Api\Model\Languageforge\Lexicon\Dto;
+use Api\Model\Shared\Mapper\MongoQueries;
+
+class LexStatsDto
+{
+    /**
+     * @param ProjectModel $project
+     * @throws \Exception
+     * @return array
+     */
+    public static function encode($project)
+    {
+        $db = MongoStore::connect($project->databaseName());
+        $num_entries = MongoQueries::countEntries($db, "lexicon");
+        $num_entries_with_pictures = MongoQueries::countEntriesWithPictures($db, "lexicon");
+        $num_unresolved_comments = MongoQueries::countUnresolvedComments($db, "lexiconComments");
+        return [
+            "num_entries" => $num_entries,
+            "num_entries_with_pictures" => $num_entries_with_pictures,
+            "num_unresolved_comments" => $num_unresolved_comments,
+        ];
+    }
+}

src/Api/Model/Shared/Mapper/MongoQueries.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace Api\Model\Shared\Mapper;
+
+class MongoQueries
+{
+    public static function countEntries($db, $collectionName)
+    {
+        $coll = $db->selectCollection($collectionName);
+        return $coll->count();
+    }
+
+    public static function countEntriesWithPictures($db, $collectionName)
+    {
+        $coll = $db->selectCollection($collectionName);
+        $query = [
+            "senses" => ['$exists' => true, '$ne' => []],
+            "senses.pictures" => ['$exists' => true, '$ne' => []],
+        ];
+        return $coll->count($query);
+    }
+
+    public static function countUnresolvedComments($db, $collectionName)
+    {
+        $coll = $db->selectCollection($collectionName);
+        $query = [
+            "status" => ['$exists' => true, '$ne' => "resolved"],
+        ];
+        return $coll->count($query);
+    }
+}

src/Api/Service/Sf.php

@@ -14,6 +14,7 @@
 use Api\Model\Languageforge\Lexicon\Dto\LexBaseViewDto;
 use Api\Model\Languageforge\Lexicon\Dto\LexDbeDto;
 use Api\Model\Languageforge\Lexicon\Dto\LexProjectDto;
+use Api\Model\Languageforge\Lexicon\Dto\LexStatsDto;
 use Api\Model\Shared\Command\ProjectCommands;
 use Api\Model\Shared\Command\SessionCommands;
 use Api\Model\Shared\Command\UserCommands;
@@ -518,7 +519,7 @@ public function lex_stats()
         $user = new UserModel($this->userId);
 
         if ($user->isMemberOfProject($this->projectId)) {
-            return LexDbeDto::encode($projectModel->id->asString(), $this->userId, 1);
+            return LexStatsDto::encode($projectModel);
         }
 
         throw new UserUnauthorizedException("User $this->userId is not a member of project $projectModel->projectCode");