GitHub duplication Fixed and Migration

controllers/auth.js

@@ -27,6 +27,7 @@ const githubAuth = (req, res, next) => {
         tokens: {
           githubAccessToken: accessToken,
         },
+        github_user_id: user.id,
       };
 
       const { userId, incompleteUserDetails } = await users.addOrUpdate(userData);

controllers/users.js

@@ -1,3 +1,5 @@
+const axios = require("axios");
+const firestore = require("../utils/firestore");
 const chaincodeQuery = require("../models/chaincodes");
 const userQuery = require("../models/users");
 const profileDiffsQuery = require("../models/profileDiffs");
@@ -593,6 +595,74 @@ const filterUsers = async (req, res) => {
   }
 };
 
+// one time script function to perform the migration - adding github_user_id field to the document
+const migrate = async (req, res) => {
+  const usersNotFound = [];
+  let countUserFound = 0;
+  let countUserNotFound = 0;
+  try {
+    // Fetch user data from GitHub API for each document in the users collection
+    // divided by 500 because firestore api guarantee that we can process in batch of 500.
+    const usersSnapshot = await firestore.collection("users").get();
+    const totalUsers = usersSnapshot.docs.length;
+    const batchCount = Math.ceil(totalUsers / 500);
+    // Create batch write operations for each batch of documents
+    for (let i = 0; i < batchCount; i++) {
+      const batchDocs = usersSnapshot.docs.slice(i * 500, (i + 1) * 500);
+      const batchWrite = firestore.batch();
+      const batchWrites = [];
+      for (const userDoc of batchDocs) {
+        const githubUsername = userDoc.data().github_id;
+        const username = userDoc.data().username;
+        const userId = userDoc.id;
+        batchWrite.update(userDoc.ref, { github_user_id: null });
+        batchWrites.push(
+          axios
+            .get(`https://api.github.com/users/${githubUsername}`, {
+              headers: {
+                "Content-Type": "application/json",
+              },
+              auth: {
+                username: config.get("githubOauth.clientId"),
+                password: config.get("githubOauth.clientSecret"),
+              },
+            })
+            .then((response) => {
+              const githubUserId = response.data.id;
+              batchWrite.update(userDoc.ref, { github_user_id: `${githubUserId}` });
+              countUserFound++;
+            })
+            .catch((error) => {
+              countUserNotFound++;
+              const invalidUsers = { userId, username, githubUsername };
+              usersNotFound.push(invalidUsers);
+              if (error.response && error.response.status === 404) {
+                logger.error("GitHub user not found", error);
+              } else {
+                logger.error("An error occurred at axios.get:", error);
+              }
+            })
+        );
+      }
+      await Promise.all(batchWrites);
+      await batchWrite.commit();
+    }
+
+    return res.status(200).json({
+      message: "Result of migration",
+      data: {
+        totalUsers: totalUsers,
+        usersUpdated: countUserFound,
+        usersNotUpdated: countUserNotFound,
+        invalidUsersDetails: usersNotFound,
+      },
+    });
+  } catch (error) {
+    logger.error(`Error while Updating all users: ${error}`);
+    return res.boom.badImplementation(INTERNAL_SERVER_ERROR);
+  }
+};
+
 const nonVerifiedDiscordUsers = async (req, res) => {
   const data = await userQuery.getDiscordUsers();
   return res.json(data);
@@ -626,6 +696,7 @@ module.exports = {
   addDefaultArchivedRole,
   getUserSkills,
   filterUsers,
+  migrate,
   verifyUserImage,
   getUserImageForVerification,
   nonVerifiedDiscordUsers,

models/users.js

@@ -44,7 +44,7 @@ const addOrUpdate = async (userData, userId = null) => {
     }
 
     // userId is null, Add or Update user
-    const user = await userModel.where("github_id", "==", userData.github_id).limit(1).get();
+    const user = await userModel.where("github_user_id", "==", userData.github_user_id).limit(1).get();
     if (!user.empty) {
       await userModel.doc(user.docs[0].id).set(userData, { merge: true });
 

routes/users.js

@@ -41,4 +41,7 @@ router.patch("/rejectDiff", authenticate, authorizeRoles([SUPERUSER]), users.rej
 router.patch("/:userId", authenticate, authorizeRoles([SUPERUSER]), users.updateUser);
 router.get("/suggestedUsers/:skillId", authenticate, authorizeRoles([SUPERUSER]), users.getSuggestedUsers);
 
+// WARNING!! - One time Script/Route to do migration.
+router.post("/migrate", authenticate, authorizeRoles([SUPERUSER]), users.migrate);
+
 module.exports = router;

test/fixtures/user/migration.js

@@ -0,0 +1,38 @@
+const githubUserInfo = require("../auth/githubUserInfo")();
+
+/**
+ * User info for GitHub auth response
+ * Multiple responses can be added to the array if required
+ *
+ * @return {Object}
+ */
+module.exports = () => {
+  return [
+    {
+      username: "ankur",
+      first_name: "Ankur",
+      last_name: "Narkhede",
+      yoe: 0,
+      img: "./img.png",
+      linkedin_id: "ankurnarkhede",
+      github_id: githubUserInfo[0].username,
+      github_display_name: githubUserInfo[0].displayName,
+      isMember: true,
+      phone: "1234567890",
+      email: "[email protected]",
+      roles: {
+        member: true,
+      },
+      tokens: {
+        githubAccessToken: "githubAccessToken",
+      },
+      status: "active",
+      profileURL: "https://abcde.com",
+      picture: {
+        publicId: "profile/mtS4DhUvNYsKqI7oCWVB/aenklfhtjldc5ytei3ar",
+        url: "https://res.cloudinary.com/realdevsquad/image/upload/v1667685133/profile/mtS4DhUvNYsKqI7oCWVB/aenklfhtjldc5ytei3ar.jpg",
+      },
+      incompleteUserDetails: false,
+    },
+  ];
+};

test/fixtures/user/user.js

@@ -17,6 +17,7 @@ module.exports = () => {
       yoe: 0,
       img: "./img.png",
       linkedin_id: "ankurnarkhede",
+      github_user_id: githubUserInfo[0].id,
       github_id: githubUserInfo[0].username,
       github_display_name: githubUserInfo[0].displayName,
       isMember: true,
@@ -44,6 +45,7 @@ module.exports = () => {
       last_name: "Bhandarkar",
       yoe: 0,
       img: "./img.png",
+      github_user_id: "23654780",
       github_id: "whydonti",
       linkedin_id: "nikhil-bhandarkar",
       twitter_id: "whatifi",
@@ -60,6 +62,7 @@ module.exports = () => {
       last_name: "Gajjewar",
       yoe: 0,
       img: "./img.png",
+      github_user_id: "23654788",
       github_id: "cartmanishere",
       linkedin_id: "pranav-gajjewar",
       twitter_id: "PGajjewar",
@@ -80,6 +83,7 @@ module.exports = () => {
       yoe: 3,
       img: "./img.png",
       linkedin_id: "sagarbajpai",
+      github_user_id: "23654790",
       github_id: "sagarbajpai",
       github_display_name: "Sagar Bajpai",
       phone: "1234567890",
@@ -105,6 +109,7 @@ module.exports = () => {
       yoe: 10,
       img: "./img.png",
       linkedin_id: "ankushdharkar",
+      github_user_id: "23654700",
       github_id: "ankushdharkar",
       github_display_name: "Ankush Dharkar",
       phone: "1234567890",
@@ -131,6 +136,7 @@ module.exports = () => {
       yoe: 0,
       img: "./img.png",
       linkedin_id: "ankitabannore",
+      github_user_id: "23654725",
       github_id: "Ankita2002-Fr",
       github_display_name: "Ankita Bannore",
       isMember: true,
@@ -155,6 +161,7 @@ module.exports = () => {
       last_name: "Chaudhari",
       yoe: 0,
       img: "./img.png",
+      github_user_id: "23654583",
       github_id: "mehulkchaudhari",
       linkedin_id: "mehulkchaudhari",
       twitter_id: "mehulkchaudhari",
@@ -180,6 +187,7 @@ module.exports = () => {
       yoe: 0,
       img: "./img.png",
       linkedin_id: "ankurnarkhede",
+      github_user_id: "23654236",
       github_id: "ankur1234",
       github_display_name: "ankur-xyz",
       phone: "1234567890",
@@ -188,6 +196,7 @@ module.exports = () => {
     {
       username: "ritvik",
       github_id: "RitvikJamwal75",
+      github_user_id: "23654123",
       first_name: "Ritvik",
       yoe: 1,
       picture: {
@@ -218,6 +227,7 @@ module.exports = () => {
       linkedin_id: "tanishqsingla",
       github_id: "tanishqsingla",
       github_display_name: "Tanishq Singla",
+      github_user_id: "26207583",
       phone: "1234567890",
       email: "[email protected]",
       tokens: {
@@ -240,6 +250,7 @@ module.exports = () => {
       linkedin_id: "darthvader",
       github_id: "darthvader",
       github_display_name: "Darth Vader",
+      github_user_id: "3188964",
       phone: "1234567890",
       email: "[email protected]",
       tokens: {

test/integration/contributions.test.js

@@ -23,17 +23,20 @@ describe("Contributions", function () {
       yoe: 0,
       img: "./img.png",
       github_id: "prakashchoudhary07",
+      github_user_id: "12345678",
       username: "prakash",
     };
     // Adding user
     await userModel.addOrUpdate(user);
     // Creating second user
     user.username = "userWithNoPrs";
     user.github_id = "userWithNoPrs";
+    user.github_user_id = "userWithNoPrs";
     await userModel.addOrUpdate(user);
     // Creating second user
     user.username = "userNoTask";
     user.github_id = "userNoTask";
+    user.github_user_id = "userNoTask";
     await userModel.addOrUpdate(user);
     //  Creating task for user
     const task = {

test/integration/tasks.test.js

@@ -229,6 +229,7 @@ describe("Tasks", function () {
       const { userId: assignedUser } = await userModel.addOrUpdate({
         github_id: "prakashchoudhary07",
         username: "user1",
+        github_user_id: "12345678",
       });
       const assignedTask = [
         {

test/integration/users.test.js

@@ -1,4 +1,5 @@
 const chai = require("chai");
+const axios = require("axios");
 const { expect } = chai;
 const chaiHttp = require("chai-http");
 
@@ -26,6 +27,7 @@ const userStatusModel = require("../../models/userStatus");
 
 const cookieName = config.get("userToken.cookieName");
 const { userPhotoVerificationData } = require("../fixtures/user/photo-verification");
+const githubUserInfo = require("../fixtures/auth/githubUserInfo")();
 const Sinon = require("sinon");
 const { INTERNAL_SERVER_ERROR } = require("../../constants/errorMessages");
 const photoVerificationModel = firestore.collection("photo-verification");
@@ -1229,4 +1231,80 @@ describe("Users", function () {
         });
     });
   });
+
+  describe("POST /users/migrate", function () {
+    let fetchStub;
+
+    beforeEach(async function () {
+      fetchStub = Sinon.stub(axios, "get");
+    });
+
+    afterEach(async function () {
+      Sinon.restore();
+    });
+
+    it("Should update the user", async function () {
+      fetchStub.resolves({
+        data: githubUserInfo[0]._json,
+      });
+      const usersMigrateResponse = await chai
+        .request(app)
+        .post(`/users/migrate`)
+        .set("Cookie", `${cookieName}=${superUserAuthToken}`);
+      expect(usersMigrateResponse).to.have.status(200);
+      expect(usersMigrateResponse.body).to.eql({
+        message: "Result of migration",
+        data: {
+          totalUsers: 2,
+          usersUpdated: 2,
+          usersNotUpdated: 0,
+          invalidUsersDetails: [],
+        },
+      });
+      const usersResponse = await chai.request(app).get(`/users`).set("cookie", `${cookieName}=${superUserAuthToken}`);
+      expect(usersResponse).to.have.status(200);
+      usersResponse.body.users.forEach((document) => {
+        expect(document).to.have.property(`github_user_id`);
+      });
+    });
+    it("Should return details of users with invalid github username", async function () {
+      fetchStub.rejects({ response: { status: 404 } });
+      const usersMigrateResponse = await chai
+        .request(app)
+        .post(`/users/migrate`)
+        .set("Cookie", `${cookieName}=${superUserAuthToken}`);
+      expect(usersMigrateResponse).to.have.status(200);
+      expect(usersMigrateResponse.body.message).to.be.equal("Result of migration");
+      expect(usersMigrateResponse.body).to.have.property("data");
+      expect(usersMigrateResponse.body.data).to.have.property("totalUsers");
+      expect(usersMigrateResponse.body.data.totalUsers).to.be.equal(2);
+      expect(usersMigrateResponse.body.data).to.have.property("usersUpdated");
+      expect(usersMigrateResponse.body.data.usersUpdated).to.be.equal(0);
+      expect(usersMigrateResponse.body.data).to.have.property("usersNotUpdated");
+      expect(usersMigrateResponse.body.data.usersNotUpdated).to.be.equal(2);
+      expect(usersMigrateResponse.body.data).to.have.property("invalidUsersDetails");
+      usersMigrateResponse.body.data.invalidUsersDetails.forEach((document) => {
+        expect(document).to.have.property("userId");
+        expect(document).to.have.property("username");
+        expect(document).to.have.property("githubUsername");
+      });
+    });
+    it("Should return unauthorized error when not logged in", function (done) {
+      chai
+        .request(app)
+        .post(`/users/migrate`)
+        .end((err, res) => {
+          if (err) {
+            return done(err);
+          }
+          expect(res).to.have.status(401);
+          expect(res.body).to.eql({
+            statusCode: 401,
+            error: "Unauthorized",
+            message: "Unauthenticated User",
+          });
+          return done();
+        });
+    });
+  });
 });