Merge pull request #1022 from khalatevarun/fix/filter-users

Fix: update fetchAllUsers to fetch only filtered users from firebase

Author: ankushdharkar

constants/firebase.js

@@ -0,0 +1,5 @@
+const BATCH_SIZE_IN_CLAUSE = 30; // since only 30 comparision values are allowed in 'in' clause
+
+module.exports = {
+  BATCH_SIZE_IN_CLAUSE,
+};

controllers/users.js

@@ -8,7 +8,7 @@ const { logType } = require("../constants/logs");
 const { fetch } = require("../utils/fetch");
 const logger = require("../utils/logger");
 const obfuscate = require("../utils/obfuscate");
-const { getPaginationLink, getFilteredUsers } = require("../utils/users");
+const { getPaginationLink, getUsernamesFromPRs } = require("../utils/users");
 const { getQualifiers } = require("../utils/helper");
 const { SOMETHING_WENT_WRONG, INTERNAL_SERVER_ERROR } = require("../constants/errorMessages");
 const { getFilteredPRsOrIssues } = require("../utils/pullRequests");
@@ -73,13 +73,13 @@ const getUsers = async (req, res) => {
     if (qualifiers?.filterBy) {
       const allPRs = await getFilteredPRsOrIssues(qualifiers);
 
-      const { allUsers } = await userQuery.fetchAllUsers();
+      const filteredUsernames = getUsernamesFromPRs(allPRs);
 
-      const filteredUsers = getFilteredUsers(allPRs, allUsers);
+      const { filterdUsersWithDetails } = await userQuery.fetchUsers(filteredUsernames);
 
       return res.json({
         message: "Users returned successfully!",
-        users: filteredUsers,
+        users: filterdUsersWithDetails,
       });
     }
 

models/users.js

@@ -8,6 +8,7 @@ const firestore = require("../utils/firestore");
 const { fetchWallet, createWallet } = require("../models/wallets");
 const { arraysHaveCommonItem } = require("../utils/array");
 const { ALLOWED_FILTER_PARAMS } = require("../constants/users");
+const { BATCH_SIZE_IN_CLAUSE } = require("../constants/firebase");
 const userModel = firestore.collection("users");
 const joinModel = firestore.collection("applicants");
 const itemModel = firestore.collection("itemTags");
@@ -178,26 +179,38 @@ const fetchPaginatedUsers = async (query) => {
   }
 };
 
-const fetchAllUsers = async () => {
+const fetchUsers = async (usernames = []) => {
   try {
     const dbQuery = userModel;
+    const filterdUsersWithDetails = [];
 
-    const snapshot = await dbQuery.get();
+    const groups = [];
+    for (let i = 0; i < usernames.length; i += BATCH_SIZE_IN_CLAUSE) {
+      groups.push(usernames.slice(i, i + BATCH_SIZE_IN_CLAUSE));
+    }
 
-    const allUsers = [];
+    // For each group, write a separate query
+    const promises = groups.map((group) => {
+      return dbQuery.where("github_id", "in", group).get();
+    });
 
-    snapshot.forEach((doc) => {
-      allUsers.push({
-        id: doc.id,
-        ...doc.data(),
-        phone: undefined,
-        email: undefined,
-        tokens: undefined,
-        chaincode: undefined,
+    const snapshots = await Promise.all(promises);
+
+    snapshots.forEach((snapshot) => {
+      snapshot.forEach((doc) => {
+        filterdUsersWithDetails.push({
+          id: doc.id,
+          ...doc.data(),
+          phone: undefined,
+          email: undefined,
+          tokens: undefined,
+          chaincode: undefined,
+        });
       });
     });
+
     return {
-      allUsers,
+      filterdUsersWithDetails,
     };
   } catch (err) {
     logger.error("Error retrieving user data", err);
@@ -392,6 +405,6 @@ module.exports = {
   getJoinData,
   getSuggestedUsers,
   fetchUserSkills,
-  fetchAllUsers,
+  fetchUsers,
   getUsersBasedOnFilter,
 };

utils/users.js

@@ -121,24 +121,19 @@ function getPaginationLink(query, cursor, documentId) {
  * @param allUsers {Array} - list of total users from firebase
  *
  */
-function getFilteredUsers(allPRs, allUsers) {
-  const uniqueUsersInOrder = new Set();
+function getUsernamesFromPRs(allPRs) {
+  const uniqueUsernamesSet = new Set();
+  const usernames = [];
 
-  allPRs?.forEach((element) => {
-    uniqueUsersInOrder.add(element.username);
-  });
-
-  const usersWithDetails = [];
-
-  uniqueUsersInOrder.forEach((username) => {
-    const userDetails = allUsers.find((user) => user.github_id === username);
-
-    if (userDetails) {
-      usersWithDetails.push(userDetails);
+  allPRs?.forEach((pr) => {
+    const username = pr?.username;
+    if (!uniqueUsernamesSet.has(username)) {
+      uniqueUsernamesSet.add(username);
+      usernames.push(username);
     }
   });
 
-  return usersWithDetails;
+  return usernames;
 }
 
 module.exports = {
@@ -148,5 +143,5 @@ module.exports = {
   getParticipantUsernames,
   getLowestLevelSkill,
   getPaginationLink,
-  getFilteredUsers,
+  getUsernamesFromPRs,
 };