From 1dc970b72070063f75b8d543557a4d863e93b968 Mon Sep 17 00:00:00 2001 From: sbuhan Date: Tue, 18 Mar 2025 14:07:03 -0400 Subject: [PATCH 1/3] add the update of users info with court 194 --- scripts/firebase-admin/updateGeneralCourt.ts | 92 ++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/firebase-admin/updateGeneralCourt.ts diff --git a/scripts/firebase-admin/updateGeneralCourt.ts b/scripts/firebase-admin/updateGeneralCourt.ts new file mode 100644 index 000000000..1b94a1772 --- /dev/null +++ b/scripts/firebase-admin/updateGeneralCourt.ts @@ -0,0 +1,92 @@ +import { Script } from "./types" +import { currentGeneralCourt } from "../../functions/src/shared" + +export const script: Script = async ({ db, args }) => { + console.log(`Update General Court ${currentGeneralCourt} in progress...`) + try { + const generalCourts = await db + .collection(`/generalCourts/${currentGeneralCourt}/members`) + .get() + console.log(`${generalCourts.size} members fetched`) + + // check if generalCourts is valid + if (generalCourts.empty) { + console.log("ERROR: No members found in the collection.") + return + } + + const courtMembersMap = new Map() + for (const member of generalCourts.docs) { + const data = member.data() + // store the member data in a map with the district as the key + courtMembersMap.set(data.content["District"], data) + } + + // prepare update + const writer = db.bulkWriter() + + console.log("Fetching all profiles ...") + const users = await db.collection("profiles").get() + console.log(`${users.size} profiles fetched`) + + for (const user of users.docs) { + let data = user.data() + console.log("User info:", user.id) + + if (data.hasOwnProperty("senator")) { + const senator = data.senator + if (courtMembersMap.has(senator.district)) { + console.log( + `User ${data.displayName} has senator ${senator.name} ${senator.id} that is in the general court 194` + ) + const senatorCurrentGC = courtMembersMap.get(senator.district) + data.senator.id = senatorCurrentGC.id + data.senator.name = senatorCurrentGC.content["Name"] + console.log( + `Updating ${senatorCurrentGC.content["Name"]} ${senatorCurrentGC.id}` + ) + + writer.update(db.doc(`/profiles/${user.id}`), { + senator: data.senator + }) + } else { + console.log(`User ${data.displayName} has ${senator.id} ${senator.name} + with UNKNOW district ${senator.district} in the general court 194`) + } + } else { + console.log("WARNING: no senator found!") + } + + if (data.hasOwnProperty("representative")) { + const representative = data.representative + if (courtMembersMap.has(representative.district)) { + console.log( + `User ${data.displayName} has representative ${representative.name} ${representative.id} that is in the general court 194` + ) + const representativeCurrentGC = courtMembersMap.get( + representative.district + ) + //console.log(`Representative ${JSON.stringify(representativeCurrentGC, null, 2)} in the general court 194`) + data.representative.id = representativeCurrentGC.id + data.representative.name = representativeCurrentGC.content["Name"] + console.log( + `Updating representative ${representativeCurrentGC.content["Name"]} ${representativeCurrentGC.id}` + ) + + writer.update(db.doc(`/profiles/${user.id}`), { + senator: data.senator + }) + } else { + console.log(`User ${data.displayName} has ${representative.id} ${representative.name} + with UNKNOW district ${representative.district}in the general court 194`) + } + } else { + console.log("WARNING: no representative found!") + } + } + await writer.close() + console.log(`Updated ${users.size} documents`) + } catch (error) { + console.error("ERROR: updating General Court failed with ", error) + } +} From 13b42da91792d06850ab4de6036249fe20fac9c5 Mon Sep 17 00:00:00 2001 From: sbuhan Date: Tue, 15 Apr 2025 21:56:21 -0400 Subject: [PATCH 2/3] new implementation and log trace --- scripts/firebase-admin/updateGeneralCourt.ts | 171 ++++++++++++++----- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/scripts/firebase-admin/updateGeneralCourt.ts b/scripts/firebase-admin/updateGeneralCourt.ts index 1b94a1772..0fd4ca00b 100644 --- a/scripts/firebase-admin/updateGeneralCourt.ts +++ b/scripts/firebase-admin/updateGeneralCourt.ts @@ -1,7 +1,19 @@ import { Script } from "./types" import { currentGeneralCourt } from "../../functions/src/shared" +import { Boolean, Optional, Record } from "runtypes" +import { Profile} from "components/db/profile/types" + +const Args = Record({ + dryRun: Optional(Boolean) +}) export const script: Script = async ({ db, args }) => { + const { dryRun } = Args.check(args) + + // if (!dryRun) { + // await profileDoc.ref.update({ nextDigestAt }) + // } + console.log(`Update General Court ${currentGeneralCourt} in progress...`) try { const generalCourts = await db @@ -19,6 +31,7 @@ export const script: Script = async ({ db, args }) => { for (const member of generalCourts.docs) { const data = member.data() // store the member data in a map with the district as the key + console.log(`${data.content["District"]}: ${data.content["Name"]} , ${data.content["MemberCode"]}`) courtMembersMap.set(data.content["District"], data) } @@ -29,64 +42,130 @@ export const script: Script = async ({ db, args }) => { const users = await db.collection("profiles").get() console.log(`${users.size} profiles fetched`) + // count updated profiles + let updatedProfiles = 0 + + let numUsersWithoutSenator = 0 + let numUsersWithUpToDateSenator = 0 + + let numUsersWithoutRepres = 0 + let numUsersWithUpToDateRepres = 0 + for (const user of users.docs) { - let data = user.data() console.log("User info:", user.id) - + let hasBeenUpdated = false + let data = user.data() as Profile if (data.hasOwnProperty("senator")) { - const senator = data.senator - if (courtMembersMap.has(senator.district)) { - console.log( - `User ${data.displayName} has senator ${senator.name} ${senator.id} that is in the general court 194` - ) - const senatorCurrentGC = courtMembersMap.get(senator.district) - data.senator.id = senatorCurrentGC.id - data.senator.name = senatorCurrentGC.content["Name"] - console.log( - `Updating ${senatorCurrentGC.content["Name"]} ${senatorCurrentGC.id}` - ) - - writer.update(db.doc(`/profiles/${user.id}`), { - senator: data.senator - }) - } else { - console.log(`User ${data.displayName} has ${senator.id} ${senator.name} - with UNKNOW district ${senator.district} in the general court 194`) - } + const updateData = updateProfileMember(data, "senator", courtMembersMap) + if (updateData) { + // update profile + if (!dryRun) { + await writer.update(user.ref, updateData) + } + hasBeenUpdated = true + numUsersWithUpToDateSenator++ + } } else { - console.log("WARNING: no senator found!") + console.log(`No senator set on profile ${user.id}!`) + numUsersWithoutSenator++ } if (data.hasOwnProperty("representative")) { - const representative = data.representative - if (courtMembersMap.has(representative.district)) { - console.log( - `User ${data.displayName} has representative ${representative.name} ${representative.id} that is in the general court 194` - ) - const representativeCurrentGC = courtMembersMap.get( - representative.district - ) - //console.log(`Representative ${JSON.stringify(representativeCurrentGC, null, 2)} in the general court 194`) - data.representative.id = representativeCurrentGC.id - data.representative.name = representativeCurrentGC.content["Name"] - console.log( - `Updating representative ${representativeCurrentGC.content["Name"]} ${representativeCurrentGC.id}` - ) - - writer.update(db.doc(`/profiles/${user.id}`), { - senator: data.senator - }) - } else { - console.log(`User ${data.displayName} has ${representative.id} ${representative.name} - with UNKNOW district ${representative.district}in the general court 194`) - } + const updateData = updateProfileMember(data, "representative", courtMembersMap) + if (updateData) { + // update profile + hasBeenUpdated = hasBeenUpdated || true + // if (!dryRun) { + // await writer.set(user.ref, {repres}) + // } + numUsersWithUpToDateRepres++ + } } else { - console.log("WARNING: no representative found!") + console.log(`No senator set on profile ${user.id}!`) + numUsersWithoutRepres++ + } + + if (hasBeenUpdated) { + updatedProfiles++ } } + + // summary of updated profiles + // numUsersWithoutSenator/numUsersWithUpToDateSenator/numUsersUpdated + displaySummaryChanges("senator", updatedProfiles, numUsersWithoutSenator, + numUsersWithUpToDateSenator) + displaySummaryChanges("representative", updatedProfiles, numUsersWithoutRepres, + numUsersWithUpToDateRepres) + await writer.close() - console.log(`Updated ${users.size} documents`) } catch (error) { console.error("ERROR: updating General Court failed with ", error) } } + + + +// return Profile update if change is needed +// return undefined if no change is needed +const updateProfileMember = ( + profileRef: Profile, + memberType: string, + currentMembersMap: Map, + ): Profile | undefined => { + + let shouldUpdate = false; + let lastMember = profileRef.representative; + if (memberType == "senator") { + lastMember = profileRef.senator; + } + + if (!lastMember) { + console.log(`User has no ${memberType}`) + return undefined + } + + if (currentMembersMap.has(lastMember.district)) { + const currentMember = currentMembersMap.get(lastMember.district) + console.log( + `User has ${memberType} ${lastMember.id} ${currentMember.id}` + ) + + if (currentMember.name != lastMember.name) { + shouldUpdate = true + } + + if (shouldUpdate) { + if (memberType == "senator") { + profileRef.senator = { + id: currentMember.id, + name: currentMember.content["Name"], + district: currentMember.content["District"] + } + } + else { + profileRef.representative = { + id: currentMember.id, + name: currentMember.content["Name"], + district: currentMember.content["District"] + } + } + console.log(`Updating ${memberType} ${currentMember.content["Name"]} ${currentMember.id}`) + return profileRef; + } + } else { + console.log(`District ${lastMember.district} ${lastMember.id} not found in the new court`) + } + return undefined +} + +const displaySummaryChanges = ( + memberType: string, + updatedProfiles: number, + numUsersWithoutMember: number, + numUsersWithUpToDateMember: number + ) => { + console.log(`Summary of ${memberType} changes`) + console.log(`Updated ${updatedProfiles} profiles`) + console.log(`Number of users without ${memberType}: ${numUsersWithoutMember}`) + console.log(`Number of users with up to date ${memberType}: ${numUsersWithUpToDateMember}`) +} \ No newline at end of file From 11be6b5377a63a488c70eb523db56acd5c16d89f Mon Sep 17 00:00:00 2001 From: sbuhan Date: Tue, 22 Apr 2025 21:19:02 -0400 Subject: [PATCH 3/3] fix bulkWriter exiting scripts --- scripts/firebase-admin/updateGeneralCourt.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/firebase-admin/updateGeneralCourt.ts b/scripts/firebase-admin/updateGeneralCourt.ts index 0fd4ca00b..b95a65273 100644 --- a/scripts/firebase-admin/updateGeneralCourt.ts +++ b/scripts/firebase-admin/updateGeneralCourt.ts @@ -56,11 +56,11 @@ export const script: Script = async ({ db, args }) => { let hasBeenUpdated = false let data = user.data() as Profile if (data.hasOwnProperty("senator")) { - const updateData = updateProfileMember(data, "senator", courtMembersMap) + let updateData = updateProfileMember(data, "senator", courtMembersMap) if (updateData) { // update profile if (!dryRun) { - await writer.update(user.ref, updateData) + writer.update(user.ref, updateData) } hasBeenUpdated = true numUsersWithUpToDateSenator++ @@ -74,14 +74,14 @@ export const script: Script = async ({ db, args }) => { const updateData = updateProfileMember(data, "representative", courtMembersMap) if (updateData) { // update profile + if (!dryRun) { + writer.update(user.ref, updateData) + } hasBeenUpdated = hasBeenUpdated || true - // if (!dryRun) { - // await writer.set(user.ref, {repres}) - // } numUsersWithUpToDateRepres++ } } else { - console.log(`No senator set on profile ${user.id}!`) + console.log(`No representative set on profile ${user.id}!`) numUsersWithoutRepres++ }