Merge remote-tracking branch 'upstream/main'

Author: efrenmarin45

.github/workflows/hfUpdate.yml

@@ -2,7 +2,7 @@ name: Update Huggingface Dataset
 
 on:
   schedule: # once daily at 3:05 pm PST (23:05 UTC)
-    - cron: '5 23 * * *'
+    - cron: "5 23 * * *"
 
 jobs:
   run:
@@ -11,20 +11,20 @@ jobs:
       run:
         working-directory: scripts/
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-    - name: Set up Python environment
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.11.x'
+      - name: Set up Python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.11.x"
 
-    - name: Install dependencies
-      run: pip install -r requirements.txt
+      - name: Install dependencies
+        run: pip install -r requirements.txt
 
-    - name: Setup environment
-      run: |
-        echo "HUGGINGFACE_LOGIN_TOKEN=${{ secrets.HUGGINGFACE_LOGIN_TOKEN }}" > .env
-
-    - name: Run script
-      run: python updateHfDataset.py
+      - name: Setup environment
+        run: |
+          echo "HUGGINGFACE_LOGIN_TOKEN=${{ secrets.HUGGINGFACE_LOGIN_TOKEN }}" > .env
+          echo "VITE_ENV=${{ secrets.VITE_ENV }}" >> .env
+      - name: Run script
+        run: python updateHfDataset.py

contributing.md CONTRIBUTING.mdcontributing.md renamed to CONTRIBUTING.md

@@ -66,7 +66,8 @@
     "redux-logger": "^3.0.6",
     "redux-saga": "^1.2.3",
     "regenerator-runtime": "^0.14.0",
-    "web-worker": "^1.2.0"
+    "web-worker": "^1.2.0",
+    "yup": "^1.6.1"
   },
   "devDependencies": {
     "@testing-library/react": "^12.1.5",

package-lock.json

@@ -8,7 +8,7 @@
 load_dotenv()
 
 # set environment as 'dev' or 'prod'
-ENV = os.getenv('VITE_ENV').upper()
+ENV = os.getenv('VITE_ENV')
 
 if ENV == 'DEV':
     HF_USERNAME = '311-Data-Dev'
@@ -101,4 +101,4 @@ def main():
     cleanUp()
 
 
-main()
+main()

package.json

@@ -1,7 +1,7 @@
 const settings = {
   map: {
     debounce: {
-      duration: 1500, // milliseconds
+      duration: 100, // milliseconds
       options: {
         leading: true, // calls function immediately when invoked (React-friendly)
         trailing: false, // calls function after last input event fired (Not React-friendly)

scripts/updateHfDataset.py

@@ -1,4 +1,5 @@
 import DataFrame from 'dataframe-js';
+import { object, string, number, date, array } from 'yup';
 
 const dataResources = {
   2019: 'pvft-t768',
@@ -12,6 +13,64 @@ export function getDataResources() {
   return dataResources;
 }
 
+const serviceRequestSchema = object({
+  actiontaken: string(),
+  address: string().min(3).max(100).nullable(),
+  addressverified: string(),
+  anonymous: string(),
+  apc: string(),
+  approximateaddress: string(),
+  assignto: string(),
+  cd: string().nullable(),
+  cdmember: string(),
+  closeddate: date(),
+  createdbyuserorganization: string(),
+  createddate: date(),
+  direction: string().nullable(),
+  housenumber: string(),
+  latitude: number().nullable(),
+  location: object({ 
+    type: string().matches(/Point/),
+    coordinates: array().of(number()).length(2)
+  }),
+  longitude: number().nullable(),
+  mobileos: string().nullable(),
+  nc: string(),
+  ncname: string(),
+  owner: string(),
+  policeprecinct: string(),
+  requestsource: string(),
+  requesttype: string(),
+  servicedate: date(),
+  srnumber: string().length(12),
+  status: string(),
+  streetname: string().max(32).nullable(),
+  suffix: string(),
+  tbmcolumn: string().length(1),
+  tbmpage: number().integer().max(999).nullable(), 
+  tbmrow: string().nullable(),
+  updateddate: date(),
+  zipcode: string().nullable(),
+}) 
+
+const srArraySchema = array().of(
+    serviceRequestSchema
+  )
+
+export async function getServiceRequests() {
+  try {
+      // Get 2024 SR data through Socrata API
+    const response = await fetch("https://data.lacity.org/resource/b7dx-7gc3.json");
+    const unvalidatedSrs = await response.json();
+    console.log(unvalidatedSrs);
+    const validatedSrs = await srArraySchema.validate(unvalidatedSrs);
+    console.log(validatedSrs);
+    return validatedSrs;
+  } catch (error) {
+    console.error('Error fetching service requests:', error);
+  }
+}
+
 export function getColorMap(discrete) {
   if (discrete) {
     return [

settings/index.js

@@ -181,4 +181,4 @@ export function shuffle(array) {
     .map(a => ({ sort: Math.random(), value: a }))
     .sort((a, b) => a.sort - b.sort)
     .map(a => a.value);
-}
+}