diff --git a/backend/models/user.model.js b/backend/models/user.model.js index 4c2005cc6..6273320ff 100644 --- a/backend/models/user.model.js +++ b/backend/models/user.model.js @@ -8,7 +8,7 @@ const userSchema = mongoose.Schema({ firstName: { type: String }, lastName: { type: String }, }, - email: { type: String, unique: true }, + email: { type: String, unique: true, lowercase: true }, accessLevel: { type: String, enum: ["user", "admin", "superadmin"], // restricts values to "user", "admin" and "superadmin" diff --git a/backend/models/user.model.test.js b/backend/models/user.model.test.js index 03d37f79e..ed802e2e9 100644 --- a/backend/models/user.model.test.js +++ b/backend/models/user.model.test.js @@ -96,6 +96,19 @@ describe('Unit tests for User Model', () => { expect(error.errors.accessLevel).toBeDefined(); }); + it('should enforce that emails are stored in lowercase', async () => { + // Create a mock user with an uppercase email + const uppercaseEmail = 'TEST@test.com'; + const mockUser = new User({ + email: uppercaseEmail, + }); + + mockUser.validate(); + // Tests + expect(mockUser.email).toBe(uppercaseEmail.toLowerCase()); + }); + + it('should pass validation with valid user data', async () => { // Create a mock user with valid data const mockUser = new User({ diff --git a/backend/scripts/python/env/.gitignore b/backend/scripts/python/env/.gitignore new file mode 100644 index 000000000..f514b74c5 --- /dev/null +++ b/backend/scripts/python/env/.gitignore @@ -0,0 +1,2 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +* diff --git a/backend/scripts/python/env/Duplicate Removal.ipynb b/backend/scripts/python/env/Duplicate Removal.ipynb new file mode 100644 index 000000000..847f9e455 --- /dev/null +++ b/backend/scripts/python/env/Duplicate Removal.ipynb @@ -0,0 +1,405 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "377dcf28-dc41-416c-85bd-03c723ac73c5", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "For dev, you must have the backend api running on your computer. For prod, please change USER_API_URL to reflect the production url.\n", + "\n", + "Please also configure the `x-custom-required-header` within your `.env` file to have the correct value." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bf6a5708-01b8-4439-b085-996a0b9309df", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import json\n", + "from dotenv import load_dotenv\n", + "import os\n", + "import re\n", + "from datetime import datetime\n", + "from functools import reduce\n", + "\n", + "load_dotenv()\n", + "custom_request_header = os.getenv(\"CUSTOM_REQUEST_HEADER\")\n", + "DATABASE_URL = os.getenv(\"DATABASE_URL\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3387bab6-f11c-47c2-9c7a-def83999f50e", + "metadata": {}, + "outputs": [], + "source": [ + "USER_API_URL = 'http://localhost:3000/api/users'\n", + "HEADERS = { \"x-customrequired-header\": custom_request_header }" + ] + }, + { + "cell_type": "markdown", + "id": "0c952dc5-c39e-4337-9043-16c1dbce38b3", + "metadata": {}, + "source": [ + "## Retrieve Users\n", + "\n", + "Retrieve a list of all users and the format it into a dictionary where users are hashed to their _id." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "2345b8e1-5601-4852-89c0-7e01f1b15e04", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a List of all users\n", + "r = requests.get(USER_API_URL, headers=HEADERS)\n", + "users = json.loads(r.content)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "0fb71fe2-9976-4d24-b308-d9f511d01192", + "metadata": {}, + "outputs": [], + "source": [ + "user_dict = {}\n", + "for user in users:\n", + " user_dict[user['_id']] = user" + ] + }, + { + "cell_type": "markdown", + "id": "e8658c5b-dc98-4954-befe-ddfc54668a25", + "metadata": {}, + "source": [ + "## Identify Capitalized Emails\n", + "\n", + "Create a function that identifies which users have capital characters in their email addresses. This function will return a dictionary of user ids hashed to the email that they all share called `duped_emails` and a set of tuples for capitalized emails addresses that don't have multiple user ids called `non_duped_capital_emails_with_ids`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8c5646cc-e708-4212-b280-d56711d71f64", + "metadata": {}, + "outputs": [], + "source": [ + "def identify_problem_users(users):\n", + " users_with_capital_emails = [user for user in users if re.compile('[A-Z]').search(user['email'])]\n", + "\n", + " potential_duplicate_emails = set([user['email'].lower() for user in users_with_capital_emails])\n", + "\n", + " problem_users = {}\n", + " for user in users:\n", + " current_email = user['email'].lower()\n", + " if current_email in potential_duplicate_emails:\n", + " if problem_users.get(current_email, None) is not None:\n", + " problem_users[current_email].append(user['_id'])\n", + " else:\n", + " problem_users[current_email] = [user['_id']]\n", + "\n", + " non_duped_capital_emails_with_ids = set([(email, problem_users[email][0]) for email in problem_users.keys() if len(problem_users[email]) == 1])\n", + "\n", + " for email, user_id in non_duped_capital_emails_with_ids:\n", + " problem_users.pop(email)\n", + "\n", + " return problem_users, non_duped_capital_emails_with_ids" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "feae3b2e-8bb8-43d2-9056-5db90b44708f", + "metadata": {}, + "outputs": [], + "source": [ + "duped_emails, non_duped_capital_emails_with_ids = identify_problem_users(users)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8f0c346-9ac5-4e05-b3d7-25e68d6cbcf7", + "metadata": {}, + "outputs": [], + "source": [ + "duped_emails" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b5d62f5-3625-4693-9471-8d49e7f4b6fe", + "metadata": {}, + "outputs": [], + "source": [ + "non_duped_capital_emails_with_ids" + ] + }, + { + "cell_type": "markdown", + "id": "c762e074-04f9-4590-bef5-f4eaac0d3ac6", + "metadata": {}, + "source": [ + "## Fixing non-duped emails\n", + "\n", + "These functions will use the API to update user documents in the database that have an email with a capitalized character. To fix all such emails, run `fix_non_duped_capital_emails(non_duped_capital_emails_with_ids)`" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "dba2cc1d-2df2-4322-a9c8-dc7351e9a1ac", + "metadata": {}, + "outputs": [], + "source": [ + "def update_user(user_id, user_data):\n", + " r = requests.patch(USER_API_URL + '/' + user_id, json=user_data, headers=HEADERS)\n", + " print(r.content)\n", + " \n", + "def fix_non_duped_capital_emails(emails_with_ids):\n", + " for email, user_id in emails_with_ids:\n", + " print(email, user_id)\n", + " update_user(user_id, {'email': email})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8c5313c-05fd-4121-87ce-b4e175a112d6", + "metadata": {}, + "outputs": [], + "source": [ + "fix_non_duped_capital_emails(non_duped_capital_emails_with_ids)" + ] + }, + { + "cell_type": "markdown", + "id": "75a6a051-8622-49d4-9508-0e6ac0526d6f", + "metadata": {}, + "source": [ + "## Removing duplicate users\n", + "\n", + "These following cells will order the userIds in the duped_emails dict from oldest to newest, merge the information from new user documents into the oldest one, update the original, and then delete the duplicates. It will also keep track of the duplicate user documents' userIds so that we can change any checkins later to have the userId of the original user document." + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "3f9e99a4-c67e-4109-913d-ee8cced7bb57", + "metadata": {}, + "outputs": [], + "source": [ + "# Sort ids for each duped email by oldest to newest\n", + "\n", + "for lowercase_email in duped_emails.keys():\n", + " duped_emails[lowercase_email].sort(key=(lambda _id: user_dict[_id]['createdDate']))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "6270f2b8-aa8d-446f-8ee8-7dcbac3a4cc7", + "metadata": {}, + "outputs": [], + "source": [ + "# This will be used later for updating checkins\n", + "ids_to_replace = {}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2e9128e-b338-4456-ae0d-8772621536dd", + "metadata": {}, + "outputs": [], + "source": [ + "def merge_users(older_id, newer_id):\n", + " canonical_user = user_dict[older_id]\n", + " duplicate_user = user_dict[newer_id]\n", + " ids_to_replace[newer_id] = older_id\n", + "\n", + " # For any list fields, combine them\n", + " list_fields = ['skillsToMatch', 'projects', 'managedProjects']\n", + " for field in list_fields:\n", + " all_array_values = set(canonical_user.get(field, []) + duplicate_user.get(field, []))\n", + " canonical_user[field] = list(all_array_values)\n", + "\n", + " # For boolean fields, set to true if either is true\n", + " bool_fields = ['textingOk', 'isActive', 'newMember']\n", + " for field in bool_fields:\n", + " canonical_user[field] = canonical_user[field] or duplicate_user[field]\n", + "\n", + " # For fields about roles, take the most recent information\n", + " take_the_newer_fields = ['currentRole', 'desiredRole']\n", + " for field in take_the_newer_fields:\n", + " if len(duplicate_user.get(field, '')) > 0:\n", + " canonical_user[field] = duplicate_user[field]\n", + "\n", + " # Take the highest access level\n", + " access_level = ['user', 'admin', 'superadmin']\n", + " highest_access_level = max(access_level.index(canonical_user['accessLevel']), access_level.index(duplicate_user['accessLevel']))\n", + " canonical_user['accessLevel'] = access_level[highest_access_level]\n", + "\n", + " # Make user that email is all lower case\n", + " canonical_user['email'] = canonical_user['email'].lower()\n", + " \n", + " \n", + " return older_id" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "78ccbbd6-7fea-4657-bf9a-850bd9e68505", + "metadata": {}, + "outputs": [], + "source": [ + "def delete_user(user_id):\n", + " r = requests.delete(USER_API_URL + '/' + user_id, headers=HEADERS)\n", + " print(r.content)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "98f86221-5f32-4a1f-8537-bb40e967b17a", + "metadata": {}, + "outputs": [], + "source": [ + "for lowercase_email in duped_emails.keys():\n", + " reduce(merge_users, duped_emails[lowercase_email])\n", + " correct_user_id = duped_emails[lower_case_email][0]\n", + " dupes = duped_emails[lower_case_email][1:]\n", + " update_user(correct_user_id, user_dict[correct_user_id])\n", + " for dupe in dupes:\n", + " delete_user(dupe)" + ] + }, + { + "cell_type": "markdown", + "id": "d57d8cac-0873-42ee-86ca-aa61e8be119f", + "metadata": {}, + "source": [ + "## Correcting Checkins\n", + "\n", + "With the following cells, we use pymongo becuase our API does not expose any endpoints for editing checkins. For each duplicate_id, we will find all checkins with that userId and replace it with the id of the original user document." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2d6d014b-71c4-44a0-8247-8d10f87a2f1a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ids_to_replace" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5f4ec365-ab7c-45fe-bfbe-b35cdbb0ec94", + "metadata": {}, + "outputs": [], + "source": [ + "from pymongo import MongoClient\n", + "client = MongoClient(DATABASE_URL)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8556cf83-a412-4ee0-bfde-0447b63d6ac4", + "metadata": {}, + "outputs": [], + "source": [ + "db = client['vrms-test']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "87d77a37-c404-4aa9-9b0a-7db74039db17", + "metadata": {}, + "outputs": [], + "source": [ + "col = db['checkins']" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "ecd00e41-1f4c-4aec-97a9-bf64a92974a4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "UpdateResult({'n': 4, 'electionId': ObjectId('7fffffff0000000000000222'), 'opTime': {'ts': Timestamp(1754280535, 23), 't': 546}, 'nModified': 4, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1754280535, 23), 'signature': {'hash': b'\\x00n\\x88\\xed\\xcc\\xb1\\x7f\\x99\\xf6@l\\xd4\\xa2N\\xb3\\xfa\\x9b\\xcd\\xec\\x7f', 'keyId': 7488330297243598876}}, 'operationTime': Timestamp(1754280535, 23), 'updatedExisting': True}, acknowledged=True)" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "col.update_many({'userId': '633b9a74d98663001f8b5c46'}, {'$set': {'userId': '5e965e554e2fc70017aa3970'}})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2db985ba-fab9-4820-9d67-ed42f8f1ba03", + "metadata": {}, + "outputs": [], + "source": [ + "for duplicate_user_id in ids_to_replace.keys():\n", + " col.update_many({'userId': duplicate_user_id}, {'$set': {'userId': ids_to_replace[duplicate_user_id]}})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/backend/scripts/python/env/README.md b/backend/scripts/python/env/README.md new file mode 100644 index 000000000..83fb65def --- /dev/null +++ b/backend/scripts/python/env/README.md @@ -0,0 +1,69 @@ +# Python Virtual Environment + +Welcome to the `scripts\python\env` folder of the VRMS backend. This folder contains a Jupyter notebook, dependencies for setting up the environment, and a `.gitignore` file for managing which files should be ignored by version control. + +## Prerequisites + +Before you begin, make sure you have Python installed on your machine. If you don't have Python installed yet, you can download and install it from the official website: + +[Download Python](https://www.python.org/downloads/) + +Once you have Python installed, you're ready to set up the virtual environment and install the necessary dependencies as described below. + +## Requirements + +Before you can run the Jupyter notebook, you will need to set up a Python virtual environment and install the required dependencies. Here's how you can do that: + +### 1. Set Up a Python Virtual Environment + +From within the `scripts` directory, run the following command to create a virtual environment: + +``` +python -m venv . +``` + +This will create a virtual environment within the current directory. + +### 2. Activate the Virtual Environment + +Once the virtual environment is created, you'll need to activate it. + +- On **Windows**, run: + + ``` + .\Scripts\activate + ``` + +- On **MacOS/Linux**, run: + + ``` + source bin/activate + ``` + +### 3. Install Dependencies + +With the virtual environment activated, you can now install the dependencies listed in `requirements.txt`: + +``` +pip install -r requirements.txt +``` + +### 4. Launch Jupyter Notebook + +After installing the required dependencies, you can start the Jupyter notebook by running the following command: + +``` +jupyter notebook +``` + +This will open the Jupyter notebook interface in your web browser, where you can navigate to and run the script. + +## .gitignore + +The `.gitignore` file in this directory is set to ignore all files, including the virtual environment, so that unnecessary files don't get committed to version control. If you wish to track changes to new files added to this directory, you will need to use a command like: + +``` +git add -f .\backend\scripts\python\env\your-file.file +``` + +where -f forces git to add and begin tracking that file. \ No newline at end of file diff --git a/backend/scripts/python/env/requirements.txt b/backend/scripts/python/env/requirements.txt new file mode 100644 index 000000000..f3c9e906d --- /dev/null +++ b/backend/scripts/python/env/requirements.txt @@ -0,0 +1,104 @@ +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==25.1.0 +arrow==1.3.0 +asttokens==3.0.0 +async-lru==2.0.5 +attrs==25.3.0 +babel==2.17.0 +beautifulsoup4==4.13.4 +bleach==6.2.0 +certifi==2025.7.14 +cffi==1.17.1 +charset-normalizer==3.4.2 +colorama==0.4.6 +comm==0.2.3 +debugpy==1.8.15 +decorator==5.2.1 +defusedxml==0.7.1 +dnspython==2.7.0 +dotenv==0.9.9 +executing==2.2.0 +fastjsonschema==2.21.1 +fqdn==1.5.1 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.10 +ipykernel==6.30.0 +ipython==9.4.0 +ipython_pygments_lexers==1.1.1 +ipywidgets==8.1.7 +isoduration==20.11.0 +jedi==0.19.2 +Jinja2==3.1.6 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.25.0 +jsonschema-specifications==2025.4.1 +jupyter==1.1.1 +jupyter-console==6.6.3 +jupyter-events==0.12.0 +jupyter-lsp==2.2.6 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.5 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_widgets==3.0.15 +lark==1.2.2 +MarkupSafe==3.0.2 +matplotlib-inline==0.1.7 +mistune==3.1.3 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +notebook==7.4.4 +notebook_shim==0.2.4 +overrides==7.7.0 +packaging==25.0 +pandocfilters==1.5.1 +parso==0.8.4 +platformdirs==4.3.8 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +psutil==7.0.0 +pure_eval==0.2.3 +pycparser==2.22 +Pygments==2.19.2 +pymongo==4.13.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-json-logger==3.3.0 +pywin32==311 +pywinpty==2.0.15 +PyYAML==6.0.2 +pyzmq==27.0.0 +referencing==0.36.2 +requests==2.32.4 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rfc3987-syntax==1.1.0 +rpds-py==0.26.0 +Send2Trash==1.8.3 +setuptools==80.9.0 +six==1.17.0 +sniffio==1.3.1 +soupsieve==2.7 +stack-data==0.6.3 +terminado==0.18.1 +tinycss2==1.4.0 +tornado==6.5.1 +traitlets==5.14.3 +types-python-dateutil==2.9.0.20250708 +typing_extensions==4.14.1 +uri-template==1.3.0 +urllib3==2.5.0 +wcwidth==0.2.13 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +widgetsnbextension==4.0.14 diff --git a/client/src/components/auth/Auth.jsx b/client/src/components/auth/Auth.jsx index e5fa196fe..b37803a06 100644 --- a/client/src/components/auth/Auth.jsx +++ b/client/src/components/auth/Auth.jsx @@ -77,7 +77,7 @@ const Auth = () => { }; function handleInputChange(e) { - const inputValue = e.currentTarget.value.toString(); + const inputValue = e.currentTarget.value.toString().toLowerCase(); validateEmail(); if (!inputValue) { setIsDisabled(true); @@ -85,7 +85,7 @@ const Auth = () => { } else { setIsDisabled(false); setIsError(false); - setEmail(e.currentTarget.value.toString()); + setEmail(e.currentTarget.value.toString().toLowerCase()); } } diff --git a/client/src/components/dashboard/AddTeamMember.jsx b/client/src/components/dashboard/AddTeamMember.jsx index 61e8143d8..d0b1fb1d4 100644 --- a/client/src/components/dashboard/AddTeamMember.jsx +++ b/client/src/components/dashboard/AddTeamMember.jsx @@ -4,7 +4,7 @@ import "../../sass/AddTeamMember.scss"; const AddTeamMember = (props) => { const [email, setEmail] = useState(""); - const handleInputChange = (e) => setEmail(e.currentTarget.value); + const handleInputChange = (e) => setEmail(e.currentTarget.value.toLowerCase()); return (
diff --git a/client/src/components/presentational/newUserForm.jsx b/client/src/components/presentational/newUserForm.jsx index 1d86acf3f..178be15fb 100644 --- a/client/src/components/presentational/newUserForm.jsx +++ b/client/src/components/presentational/newUserForm.jsx @@ -63,7 +63,7 @@ const NewUserForm = (props) => { type="email" name="email" placeholder="Email Address" - value={props.formInput.email.toString()} + value={props.formInput.email.toString().toLowerCase()} onChange={props.handleInputChange} aria-label="Email Address" required diff --git a/client/src/components/presentational/returnUserForm.jsx b/client/src/components/presentational/returnUserForm.jsx index fe177a241..53efd4815 100644 --- a/client/src/components/presentational/returnUserForm.jsx +++ b/client/src/components/presentational/returnUserForm.jsx @@ -30,7 +30,7 @@ const ReturnUserForm = (props) => { name="email" placeholder="Email Address" required="required" - value={props.formInput.email.toString()} + value={props.formInput.email.toString().toLowerCase()} onChange={props.handleInputChange} aria-label="Email Address" data-test="input-email" diff --git a/client/src/pages/CheckInForm.jsx b/client/src/pages/CheckInForm.jsx index 4c8ba4f72..b6c62ed54 100644 --- a/client/src/pages/CheckInForm.jsx +++ b/client/src/pages/CheckInForm.jsx @@ -308,7 +308,7 @@ const CheckInForm = (props) => { const currMonth = parseInt(moment().format('MM')); const yearJoined = parseInt(year); const monthJoined = parseInt(moment(month + ' 9, 2020').format('MM')); - + if ( yearJoined > currYear || (yearJoined === currYear && monthJoined > currMonth)