Improve Readme + Logfire

.gitignore

@@ -10,3 +10,5 @@ nginx/prod/keys/
 activate.prod.*
 activate.beta.*
 .pytest_cache/
+/.logfire/
+.env

Makefile

@@ -31,3 +31,11 @@ build:
 .PHONY: prod-push
 prod-push:
 	git push heroku `git rev-parse --abbrev-ref HEAD`:master
+
+.PHONY: reset-db
+reset-db:
+.PHONY: reset-db
+reset-db:
+	psql -h localhost -U postgres -c "DROP DATABASE IF EXISTS socket"
+	psql -h localhost -U postgres -c "CREATE DATABASE socket"
+	python tcsocket/run.py resetdb --no-input

README.md

@@ -1,17 +1,114 @@
-socket-server
-=============
+# Socket Server
 
 ![Build Status](https://github.com/tutorcruncher/socket-server/workflows/CI/badge.svg)
 [![codecov](https://codecov.io/gh/tutorcruncher/socket-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tutorcruncher/socket-server)
 
 Backend application for [TutorCruncher's](https://tutorcruncher.com) web integration.
+## Setup and Run
 
-# LICENSE
+To set up and run this project, follow these steps:
 
-Copyright TutorCruncher ltd. 2017 - 2022.
-All rights reserved.
+1. **Clone the repository:**
+   ```sh
+   git clone [email protected]:tutorcruncher/socket-server.git
+   cd socket-server
+   ```
+
+2. **Install dependencies:**
+   ```sh
+   make install
+   ```
+
+3. **Reset the database:**
+  ```sh
+  make reset-db
+  ```
+
+4. **Run the worker:**
+   ```sh
+   python tcsocket/run.py worker
+   ```
+
+5. **Run the web server:**
+   ```sh
+   python tcsocket/run.py web
+   ```
+
+**Note:** You might have to run this with `sudo` if you are not in the `docker` group.
+
+## Environment Variables
+
+The environment variables for this project are:
+
+- `BIND_IP`: The IP address to bind the web server to. Default is `127.0.0.1`.
+- `PORT`: The port number to bind the web server to. Default is `8000`.
+- `DYNO`: Used to infer whether to run the web server or worker. If it starts with `web`, the web server will run; otherwise, the worker will run.
+- `DATABASE_URL`: The URL for the database connection.
+- `REDIS_URL`: The URL for the Redis connection.
+
+You can set these environment variables in your shell or in a `.env` file. Here is an example of how to set them in a `.env` file:
+
+```sh
+BIND_IP=127.0.0.1
+PORT=8000
+DYNO=web.1
+DATABASE_URL=postgres://postgres:[email protected]:5432/socket_test
+REDIS_URL=redis://localhost:6379/0
+```
+
+## Commands
+
+- **Run the application:**
+  ```sh
+  python tcsocket/run.py auto
+  ```
+
+- **Reset the database:**
+  ```sh
+  make reset-db
+  ```
+
+- **Open an IPython shell:**
+  ```sh
+  python tcsocket/run.py shell
+  ```
 
-## Deploying
+- **Run a patch script:**
+  ```sh
+  python tcsocket/run.py patch --live <patch_name>
+  ```
+- **Format the code:**
+   ```sh
+   make format
+   ```
+
+- **Lint the code:**
+   ```sh
+   make lint
+   ```
+
+- **Run tests:**
+   ```sh
+   make test
+   ```
+
+## Docker
+
+The project includes a `Dockerfile` for building a Docker image. To build the Docker image, run:
+
+```sh
+make build
+```
+
+## Deployment
+
+To deploy the project to Heroku, use the following command:
+
+```sh
+make prod-push
+```
+
+or
 
 To deploy socket-server, please create a new tag/release, then run the following command:
 
@@ -20,3 +117,8 @@ git push heroku master
 ```
 
 **Make sure you have checked out master and pulled all the recent changes.**
+
+## License
+
+Copyright TutorCruncher ltd. 2017 - 2022.
+All rights reserved.

tcsocket/app/index.html

@@ -50,7 +50,7 @@
 </section>
 <section>
   For help integrating with socket, see
-  <a href="https://help.tutorcruncher.com/tc-socket">help.tutorcruncher.com/tc-socket</a>.
+  <a href="https://help.tutorcruncher.com/en/articles/8255881-getting-started-with-tutorcruncher-socket">https://help.tutorcruncher.com/en/articles/8255881-getting-started-with-tutorcruncher-socket</a>.
 </section>
 </body>
 </html>

tcsocket/app/logs.py

@@ -25,11 +25,13 @@ def setup_logging(verbose: bool = False):
                 'release': os.getenv('COMMIT', None),
                 'name': os.getenv('SERVER_NAME', '-'),
             },
+            'logfire': {'class': 'logfire.integrations.logging.LogfireLoggingHandler'},
         },
         'loggers': {
-            'socket': {'handlers': ['socket', 'sentry'], 'level': log_level},
-            'gunicorn.error': {'handlers': ['sentry'], 'level': 'ERROR'},
-            'arq': {'handlers': ['socket', 'sentry'], 'level': log_level},
+            'socket': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level},
+            'gunicorn.error': {'handlers': ['sentry', 'logfire'], 'level': 'ERROR'},
+            'arq': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level},
+            'aiohttp': {'handlers': ['logfire'], 'level': log_level},
         },
     }
     logging.config.dictConfig(config)

tcsocket/app/main.py

@@ -2,6 +2,7 @@
 import re
 from html import escape
 
+import logfire
 from aiohttp import ClientSession, web
 from aiopg.sa import create_engine
 from arq import create_pool
@@ -32,13 +33,22 @@ async def startup(app: web.Application):
         redis=redis,
         session=ClientSession(),
     )
+    if bool(settings.logfire_token):
+        logfire.configure(
+            service_name='socket-server',
+            token=settings.logfire_token,
+            send_to_logfire=True,
+            console=False,
+        )
+        logfire.instrument_pydantic(app)
+        logfire.instrument_requests()
+        logfire.instrument_aiohttp_client()
 
 
 async def cleanup(app: web.Application):
     app['pg_engine'].close()
     await app['pg_engine'].wait_closed()
-    app['redis'].close()
-    await app['redis'].wait_closed()
+    await app['redis'].close()
     await app['session'].close()
 
 

tcsocket/app/settings.py

@@ -3,7 +3,8 @@
 from urllib.parse import urlparse
 
 from arq.connections import RedisSettings
-from pydantic import BaseSettings, validator
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings
 
 THIS_DIR = Path(__file__).parent
 BASE_DIR = THIS_DIR.parent
@@ -14,22 +15,24 @@ class Settings(BaseSettings):
     redis_settings: RedisSettings = 'redis://localhost:6379'
     redis_database: int = 0
 
-    master_key = b'this is a secret'
+    master_key: bytes = Field(default=b'this is a secret', env='MASTER_KEY')
 
     aws_access_key: Optional[str] = 'testing'
     aws_secret_key: Optional[str] = 'testing'
     aws_bucket_name: str = 'socket-images-beta.tutorcruncher.com'
-    tc_api_root = 'https://secure.tutorcruncher.com/api'
-    grecaptcha_secret = 'required secret for google recaptcha'
-    grecaptcha_url = 'https://www.google.com/recaptcha/api/siteverify'
-    geocoding_url = 'https://maps.googleapis.com/maps/api/geocode/json'
-    geocoding_key = 'required secret for google geocoding'
+    tc_api_root: str = 'https://secure.tutorcruncher.com/api'
+    grecaptcha_secret: str = 'required secret for google recaptcha'
+    grecaptcha_url: str = 'https://www.google.com/recaptcha/api/siteverify'
+    geocoding_url: str = 'https://maps.googleapis.com/maps/api/geocode/json'
+    geocoding_key: str = 'required secret for google geocoding'
 
-    tc_contractors_endpoint = '/public_contractors/'
-    tc_enquiry_endpoint = '/enquiry/'
-    tc_book_apt_endpoint = '/recipient_appointments/'
+    tc_contractors_endpoint: str = '/public_contractors/'
+    tc_enquiry_endpoint: str = '/enquiry/'
+    tc_book_apt_endpoint: str = '/recipient_appointments/'
 
-    @validator('redis_settings', always=True, pre=True)
+    logfire_token: Optional[str] = ''
+
+    @field_validator('redis_settings', mode='before')
     def parse_redis_settings(cls, v):
         conf = urlparse(v)
         return RedisSettings(
@@ -68,12 +71,5 @@ def pg_port(self):
         return self._pg_dsn_parsed.port
 
     class Config:
-        fields = {
-            'port': {'env': 'PORT'},
-            'database_url': {'env': 'DATABASE_URL'},
-            'redis_settings': {'env': 'REDISCLOUD_URL'},
-            'tc_api_root': {'env': 'TC_API_ROOT'},
-            'aws_access_key': {'env': 'AWS_ACCESS_KEY'},
-            'aws_secret_key': {'env': 'AWS_SECRET_KEY'},
-            'aws_bucket_name': {'env': 'AWS_BUCKET_NAME'},
-        }
+        env_prefix = ''
+        env_file = '.env'

tcsocket/app/validation.py

@@ -1,9 +1,13 @@
+import logging
 from datetime import datetime
 from enum import Enum, unique
 from secrets import token_hex
 from typing import Any, List, Optional
 
-from pydantic import BaseModel, EmailStr, NoneStr, constr, validator
+from pydantic import AliasPath, BaseModel, EmailStr, Field, constr, validator
+
+logger = logging.getLogger('socket')
+
 
 EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date'
 
@@ -132,29 +136,25 @@ class EATypeEnum(str, Enum):
 class ContractorModel(BaseModel):
     id: int
     deleted: bool = False
-    first_name: constr(max_length=255) = None
-    last_name: constr(max_length=255) = None
-    town: constr(max_length=63) = None
-    country: constr(max_length=63) = None
-    last_updated: datetime = None
-    photo: NoneStr = None
-    review_rating: float = None
+    first_name: Optional[constr(max_length=255)] = None
+    last_name: Optional[constr(max_length=255)] = None
+    town: Optional[constr(max_length=63)] = None
+    country: Optional[constr(max_length=63)] = None
+    last_updated: Optional[datetime] = Field(validation_alias=AliasPath('release_timestamp'))
+    photo: Optional[str] = None
+    review_rating: Optional[float] = None
     review_duration: int = None
 
-    @validator('last_updated', pre=True, always=True)
-    def set_last_updated(cls, v):
-        return v or datetime(2016, 1, 1)
-
     class LatitudeModel(BaseModel):
         latitude: Optional[float] = None
         longitude: Optional[float] = None
 
-    location: LatitudeModel = None
-    extra_attributes: List[ExtraAttributeModel] = []
+    location: Optional[LatitudeModel] = None
+    extra_attributes: List['ExtraAttributeModel'] = []
 
     class SkillModel(BaseModel):
         subject: str
-        subject_id: str
+        subject_id: Optional[int] = None
         category: str
         qual_level: str
         qual_level_id: int
@@ -210,7 +210,7 @@ class BookingModel(BaseModel):
     student_name: str = ''
 
     @validator('student_name', always=True)
-    def check_name_or_id(cls, v, values, **kwargs):
+    def check_name_or_id(cls, v, values):
         if v == '' and values['student_id'] is None:
             raise ValueError('either student_id or student_name is required')
         return v

tcsocket/app/views/appointments.py

@@ -4,9 +4,9 @@
 from datetime import datetime, timezone
 from operator import attrgetter
 from secrets import compare_digest
-from typing import Dict
+from typing import Dict, Protocol
 
-from pydantic import BaseModel, Protocol, ValidationError, validator
+from pydantic import BaseModel, ValidationError, field_validator
 from sqlalchemy import distinct, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.sql import and_, functions as sql_f
@@ -270,18 +270,20 @@ class SSOData(BaseModel):
     expires: datetime
     key: str
 
-    @validator('role_type')
+    @field_validator('role_type')
     def check_role_type(cls, v):
         if v != 'Client':
             raise ValueError('must be "Client"')
+        return v
 
-    class Config:
-        fields = {
+    model_config = {
+        'alias_generator': {
             'role_type': 'rt',
             'name': 'nm',
             'students': 'srs',
             'expires': 'exp',
         }
+    }
 
 
 def _get_sso_data(request, company) -> SSOData:

tcsocket/requirements.txt

@@ -3,17 +3,19 @@ aiodns==3.0.0
 aiohttp==3.8.1
 aiopg==1.3.4
 aioredis==1.3.1
-arq==0.22
+arq==0.25.0
 boto3==1.24.57
 cchardet==2.1.7
 gunicorn==20.1.0
 python-dateutil==2.8.2
 pillow==9.2.0
-pydantic[email]==1.9.1
+pydantic[email]==2.7.0
 raven==6.10.0
 requests==2.28.1
 uvloop==0.16.0
 ipython==8.4.0
 pgcli==3.4.1
 ipython-sql==0.4.1
 yarl==1.8.1
+logfire[requests, aiohttp]~=1.3.0
+pydantic-settings~=2.6.0