diff --git a/.gitignore b/.gitignore index fc721fd..6ac9897 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ nginx/prod/keys/ activate.prod.* activate.beta.* .pytest_cache/ +/.logfire/ +.env diff --git a/Makefile b/Makefile index 68ba11e..78a9c8c 100644 --- a/Makefile +++ b/Makefile @@ -31,3 +31,11 @@ build: .PHONY: prod-push prod-push: git push heroku `git rev-parse --abbrev-ref HEAD`:master + +.PHONY: reset-db +reset-db: +.PHONY: reset-db +reset-db: + psql -h localhost -U postgres -c "DROP DATABASE IF EXISTS socket" + psql -h localhost -U postgres -c "CREATE DATABASE socket" + python tcsocket/run.py resetdb --no-input diff --git a/README.md b/README.md index f9ac598..255c86b 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,114 @@ -socket-server -============= +# Socket Server ![Build Status](https://github.com/tutorcruncher/socket-server/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/tutorcruncher/socket-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tutorcruncher/socket-server) Backend application for [TutorCruncher's](https://tutorcruncher.com) web integration. +## Setup and Run -# LICENSE +To set up and run this project, follow these steps: -Copyright TutorCruncher ltd. 2017 - 2022. -All rights reserved. +1. **Clone the repository:** + ```sh + git clone git@github.com:tutorcruncher/socket-server.git + cd socket-server + ``` + +2. **Install dependencies:** + ```sh + make install + ``` + +3. **Reset the database:** + ```sh + make reset-db + ``` + +4. **Run the worker:** + ```sh + python tcsocket/run.py worker + ``` + +5. **Run the web server:** + ```sh + python tcsocket/run.py web + ``` + +**Note:** You might have to run this with `sudo` if you are not in the `docker` group. + +## Environment Variables + +The environment variables for this project are: + +- `BIND_IP`: The IP address to bind the web server to. Default is `127.0.0.1`. +- `PORT`: The port number to bind the web server to. Default is `8000`. +- `DYNO`: Used to infer whether to run the web server or worker. If it starts with `web`, the web server will run; otherwise, the worker will run. +- `DATABASE_URL`: The URL for the database connection. +- `REDIS_URL`: The URL for the Redis connection. + +You can set these environment variables in your shell or in a `.env` file. Here is an example of how to set them in a `.env` file: + +```sh +BIND_IP=127.0.0.1 +PORT=8000 +DYNO=web.1 +DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/socket_test +REDIS_URL=redis://localhost:6379/0 +``` + +## Commands + +- **Run the application:** + ```sh + python tcsocket/run.py auto + ``` + +- **Reset the database:** + ```sh + make reset-db + ``` + +- **Open an IPython shell:** + ```sh + python tcsocket/run.py shell + ``` -## Deploying +- **Run a patch script:** + ```sh + python tcsocket/run.py patch --live + ``` +- **Format the code:** + ```sh + make format + ``` + +- **Lint the code:** + ```sh + make lint + ``` + +- **Run tests:** + ```sh + make test + ``` + +## Docker + +The project includes a `Dockerfile` for building a Docker image. To build the Docker image, run: + +```sh +make build +``` + +## Deployment + +To deploy the project to Heroku, use the following command: + +```sh +make prod-push +``` + +or To deploy socket-server, please create a new tag/release, then run the following command: @@ -20,3 +117,8 @@ git push heroku master ``` **Make sure you have checked out master and pulled all the recent changes.** + +## License + +Copyright TutorCruncher ltd. 2017 - 2022. +All rights reserved. diff --git a/tcsocket/app/index.html b/tcsocket/app/index.html index ce99a48..8f1cbee 100644 --- a/tcsocket/app/index.html +++ b/tcsocket/app/index.html @@ -50,7 +50,7 @@
For help integrating with socket, see - help.tutorcruncher.com/tc-socket. + https://help.tutorcruncher.com/en/articles/8255881-getting-started-with-tutorcruncher-socket.
diff --git a/tcsocket/app/logs.py b/tcsocket/app/logs.py index b9e821a..1f2315f 100644 --- a/tcsocket/app/logs.py +++ b/tcsocket/app/logs.py @@ -25,11 +25,13 @@ def setup_logging(verbose: bool = False): 'release': os.getenv('COMMIT', None), 'name': os.getenv('SERVER_NAME', '-'), }, + 'logfire': {'class': 'logfire.integrations.logging.LogfireLoggingHandler'}, }, 'loggers': { - 'socket': {'handlers': ['socket', 'sentry'], 'level': log_level}, - 'gunicorn.error': {'handlers': ['sentry'], 'level': 'ERROR'}, - 'arq': {'handlers': ['socket', 'sentry'], 'level': log_level}, + 'socket': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level}, + 'gunicorn.error': {'handlers': ['sentry', 'logfire'], 'level': 'ERROR'}, + 'arq': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level}, + 'aiohttp': {'handlers': ['logfire'], 'level': log_level}, }, } logging.config.dictConfig(config) diff --git a/tcsocket/app/main.py b/tcsocket/app/main.py index b3ebd64..a054c76 100644 --- a/tcsocket/app/main.py +++ b/tcsocket/app/main.py @@ -2,6 +2,7 @@ import re from html import escape +import logfire from aiohttp import ClientSession, web from aiopg.sa import create_engine from arq import create_pool @@ -32,13 +33,22 @@ async def startup(app: web.Application): redis=redis, session=ClientSession(), ) + if bool(settings.logfire_token): + logfire.configure( + service_name='socket-server', + token=settings.logfire_token, + send_to_logfire=True, + console=False, + ) + logfire.instrument_pydantic(app) + logfire.instrument_requests() + logfire.instrument_aiohttp_client() async def cleanup(app: web.Application): app['pg_engine'].close() await app['pg_engine'].wait_closed() - app['redis'].close() - await app['redis'].wait_closed() + await app['redis'].close() await app['session'].close() diff --git a/tcsocket/app/settings.py b/tcsocket/app/settings.py index 801f554..1166180 100644 --- a/tcsocket/app/settings.py +++ b/tcsocket/app/settings.py @@ -3,7 +3,8 @@ from urllib.parse import urlparse from arq.connections import RedisSettings -from pydantic import BaseSettings, validator +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings THIS_DIR = Path(__file__).parent BASE_DIR = THIS_DIR.parent @@ -14,22 +15,24 @@ class Settings(BaseSettings): redis_settings: RedisSettings = 'redis://localhost:6379' redis_database: int = 0 - master_key = b'this is a secret' + master_key: bytes = Field(default=b'this is a secret', env='MASTER_KEY') aws_access_key: Optional[str] = 'testing' aws_secret_key: Optional[str] = 'testing' aws_bucket_name: str = 'socket-images-beta.tutorcruncher.com' - tc_api_root = 'https://secure.tutorcruncher.com/api' - grecaptcha_secret = 'required secret for google recaptcha' - grecaptcha_url = 'https://www.google.com/recaptcha/api/siteverify' - geocoding_url = 'https://maps.googleapis.com/maps/api/geocode/json' - geocoding_key = 'required secret for google geocoding' + tc_api_root: str = 'https://secure.tutorcruncher.com/api' + grecaptcha_secret: str = 'required secret for google recaptcha' + grecaptcha_url: str = 'https://www.google.com/recaptcha/api/siteverify' + geocoding_url: str = 'https://maps.googleapis.com/maps/api/geocode/json' + geocoding_key: str = 'required secret for google geocoding' - tc_contractors_endpoint = '/public_contractors/' - tc_enquiry_endpoint = '/enquiry/' - tc_book_apt_endpoint = '/recipient_appointments/' + tc_contractors_endpoint: str = '/public_contractors/' + tc_enquiry_endpoint: str = '/enquiry/' + tc_book_apt_endpoint: str = '/recipient_appointments/' - @validator('redis_settings', always=True, pre=True) + logfire_token: Optional[str] = '' + + @field_validator('redis_settings', mode='before') def parse_redis_settings(cls, v): conf = urlparse(v) return RedisSettings( @@ -68,12 +71,5 @@ def pg_port(self): return self._pg_dsn_parsed.port class Config: - fields = { - 'port': {'env': 'PORT'}, - 'database_url': {'env': 'DATABASE_URL'}, - 'redis_settings': {'env': 'REDISCLOUD_URL'}, - 'tc_api_root': {'env': 'TC_API_ROOT'}, - 'aws_access_key': {'env': 'AWS_ACCESS_KEY'}, - 'aws_secret_key': {'env': 'AWS_SECRET_KEY'}, - 'aws_bucket_name': {'env': 'AWS_BUCKET_NAME'}, - } + env_prefix = '' + env_file = '.env' diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 18f9519..f7df94e 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -1,9 +1,13 @@ +import logging from datetime import datetime from enum import Enum, unique from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, NoneStr, constr, validator +from pydantic import AliasPath, BaseModel, EmailStr, Field, constr, validator + +logger = logging.getLogger('socket') + EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' @@ -132,29 +136,25 @@ class EATypeEnum(str, Enum): class ContractorModel(BaseModel): id: int deleted: bool = False - first_name: constr(max_length=255) = None - last_name: constr(max_length=255) = None - town: constr(max_length=63) = None - country: constr(max_length=63) = None - last_updated: datetime = None - photo: NoneStr = None - review_rating: float = None + first_name: Optional[constr(max_length=255)] = None + last_name: Optional[constr(max_length=255)] = None + town: Optional[constr(max_length=63)] = None + country: Optional[constr(max_length=63)] = None + last_updated: Optional[datetime] = Field(validation_alias=AliasPath('release_timestamp')) + photo: Optional[str] = None + review_rating: Optional[float] = None review_duration: int = None - @validator('last_updated', pre=True, always=True) - def set_last_updated(cls, v): - return v or datetime(2016, 1, 1) - class LatitudeModel(BaseModel): latitude: Optional[float] = None longitude: Optional[float] = None - location: LatitudeModel = None - extra_attributes: List[ExtraAttributeModel] = [] + location: Optional[LatitudeModel] = None + extra_attributes: List['ExtraAttributeModel'] = [] class SkillModel(BaseModel): subject: str - subject_id: str + subject_id: Optional[int] = None category: str qual_level: str qual_level_id: int @@ -210,7 +210,7 @@ class BookingModel(BaseModel): student_name: str = '' @validator('student_name', always=True) - def check_name_or_id(cls, v, values, **kwargs): + def check_name_or_id(cls, v, values): if v == '' and values['student_id'] is None: raise ValueError('either student_id or student_name is required') return v diff --git a/tcsocket/app/views/appointments.py b/tcsocket/app/views/appointments.py index 651eb0c..8be4996 100644 --- a/tcsocket/app/views/appointments.py +++ b/tcsocket/app/views/appointments.py @@ -4,9 +4,9 @@ from datetime import datetime, timezone from operator import attrgetter from secrets import compare_digest -from typing import Dict +from typing import Dict, Protocol -from pydantic import BaseModel, Protocol, ValidationError, validator +from pydantic import BaseModel, ValidationError, field_validator from sqlalchemy import distinct, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql import and_, functions as sql_f @@ -270,18 +270,20 @@ class SSOData(BaseModel): expires: datetime key: str - @validator('role_type') + @field_validator('role_type') def check_role_type(cls, v): if v != 'Client': raise ValueError('must be "Client"') + return v - class Config: - fields = { + model_config = { + 'alias_generator': { 'role_type': 'rt', 'name': 'nm', 'students': 'srs', 'expires': 'exp', } + } def _get_sso_data(request, company) -> SSOData: diff --git a/tcsocket/requirements.txt b/tcsocket/requirements.txt index 631165e..2e9af94 100644 --- a/tcsocket/requirements.txt +++ b/tcsocket/requirements.txt @@ -3,13 +3,13 @@ aiodns==3.0.0 aiohttp==3.8.1 aiopg==1.3.4 aioredis==1.3.1 -arq==0.22 +arq==0.25.0 boto3==1.24.57 cchardet==2.1.7 gunicorn==20.1.0 python-dateutil==2.8.2 pillow==9.2.0 -pydantic[email]==1.9.1 +pydantic[email]==2.7.0 raven==6.10.0 requests==2.28.1 uvloop==0.16.0 @@ -17,3 +17,5 @@ ipython==8.4.0 pgcli==3.4.1 ipython-sql==0.4.1 yarl==1.8.1 +logfire[requests, aiohttp]~=1.3.0 +pydantic-settings~=2.6.0 diff --git a/tcsocket/run.py b/tcsocket/run.py index d5f30a4..cbe70ca 100755 --- a/tcsocket/run.py +++ b/tcsocket/run.py @@ -35,6 +35,7 @@ def check_app(): logger.info('app started and stopped successfully, apparently configured correctly') +@cli.command() def web(): """ Serve the application @@ -46,7 +47,9 @@ def web(): check_app() - bind = os.getenv('BIND_IP', '127.0.0.1') + f":{os.getenv('PORT', '8000')}" + bind_ip = os.getenv('BIND_IP', '127.0.0.1') + port = os.getenv('PORT', '8000') + bind = f"{bind_ip}:{port}" logger.info('Starting Web, binding to %s', bind) config = dict( @@ -67,8 +70,10 @@ def load(self): logger.info('starting gunicorn...') Application().run() + logger.info('Web server running at %s on port %s', bind_ip, port) +@cli.command() def worker(): """ Run the worker @@ -79,21 +84,26 @@ def worker(): @cli.command() -def auto(): +@click.pass_context +def auto(ctx): port_env = os.getenv('PORT') dyno_env = os.getenv('DYNO') if dyno_env: logger.info('using environment variable DYNO=%r to infer command', dyno_env) if dyno_env.lower().startswith('web'): - web() + logger.info('Running as web server') + ctx.invoke(web) else: - worker() + logger.info('Running as worker') + ctx.invoke(worker) elif port_env and port_env.isdigit(): logger.info('using environment variable PORT=%s to infer command as web', port_env) - web() + logger.info('Running as web server') + ctx.invoke(web) else: logger.info('no environment variable found to infer command, assuming worker') - worker() + logger.info('Running as worker') + ctx.invoke(worker) @cli.command()