Skip to content

Commit baa4ef1

Browse files
simcaxsimcax
authored
Merge pull request #20 from simcax/database_support
Initial code for db support, new frontpage layout and lf-info integration
2 parents 05a3bd6 + 3f31493 commit baa4ef1

32 files changed

+1275
-444
lines changed

‎.github/workflows/fly-review.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,5 @@ jobs:
5959
API_PASSWORD=${{ secrets.API_PASSWORD }}
6060
ENVIRONMENT_NAME=${{ secrets.ENVIRONMENT_NAME }}
6161
REDIS_HOST=${{ secrets.REDIS_HOST }}
62-
SESSION_COOKIE_DOMAIN=${{ steps.modify-url.outputs.url }}
62+
SESSION_COOKIE_DOMAIN=${{ steps.modify-url.outputs.url }}
63+
DOORCOUNT_URL=${{ secrets.DOORCOUNT_URL}}

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
# env files
22
.env
33

4+
# Div. files
5+
*.log
6+
local
7+
48
# IDEs
59
.vscode
610

‎docs/database.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Database structure
2+
3+
## Tables
4+
|tablename|content|
5+
|---------|-------|
6+
|index|Contains the overall index for building the navigation|
7+
|pages|Table holding information about the page contenst|
8+
|md|This table contains the actual markdown, which makes up a pages content|
9+
10+
## index
11+
12+
|field|
13+
|-----|
14+
|id|row id|
15+
|parent_id|id of the parent page (top menu item)|
16+
|sub_page_id|id of the sub_page|
17+
18+
## pages
19+
20+
|field|content|
21+
|-----|-------|
22+
|id|row id|
23+
|title|title shown in the menu|
24+
|url|url the page goes to when clicked|
25+
|sub_page|bool - is it a sub_page?|
26+
27+
## md
28+
29+
|field|content|
30+
|-----|-------|
31+
|id|row id|
32+
|page_id|id of the pages row|
33+
|md|markdown content|

‎lfweb/database/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"""Database module for the LFWeb project."""
2+
3+
from .connection import GetDbConnection # noqa: F401

‎lfweb/database/connection.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""Module for handling the database connection"""
2+
3+
import os
4+
from typing import Any
5+
6+
import psycopg2 as psycopg
7+
from loguru import logger
8+
from pydantic import BaseModel, model_validator
9+
10+
11+
class DbConnectionCredentials(BaseModel):
12+
"""Class for defining the database connection credentials"""
13+
14+
host: str
15+
port: str
16+
username: str
17+
password: str
18+
database: str
19+
db_uri: str
20+
21+
@model_validator(mode="after")
22+
@classmethod
23+
def set_credentials_from_env(cls, values: Any) -> Any:
24+
"""Set the database connection credentials from the environment"""
25+
values.host = os.getenv("DB_HOST")
26+
values.port = os.getenv("DB_PORT")
27+
values.username = os.getenv("DB_USERNAME")
28+
values.password = os.getenv("DB_PASSWORD")
29+
values.database = os.getenv("DB_NAME")
30+
values.db_uri = os.getenv("DB_URI")
31+
return values
32+
33+
34+
class GetDbConnection:
35+
"""Class for handling the database connection as a context manager"""
36+
37+
host = ""
38+
port = ""
39+
username = ""
40+
password = ""
41+
database = ""
42+
connection = None
43+
44+
def __init__(self, database_creds: DbConnectionCredentials) -> None:
45+
"""Initialize the Database object"""
46+
self.host = database_creds.host
47+
self.port = database_creds.port
48+
self.username = database_creds.username
49+
self.password = database_creds.password
50+
self.database = database_creds.database
51+
52+
def __enter__(self):
53+
logger.info(f"Connecting to database {self.database}")
54+
self.connection = self._get_connection()
55+
return self.connection
56+
57+
def __exit__(self, exc_type, exc_value, traceback):
58+
self.connection.close()
59+
60+
def _get_connection(self, no_db=False):
61+
"""Get a connection to the database"""
62+
return psycopg.connect(
63+
host=self.host,
64+
port=self.port,
65+
user=self.username,
66+
password=self.password,
67+
dbname=self.database if not no_db else None,
68+
)

‎lfweb/database/db_migration.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import os
2+
3+
from sqlalchemy import (
4+
Boolean,
5+
Column,
6+
ForeignKey,
7+
Integer,
8+
MetaData,
9+
String,
10+
Table,
11+
Text,
12+
create_engine,
13+
)
14+
from sqlalchemy_utils import create_database, database_exists
15+
16+
17+
class DatabaseMigration:
18+
"""Class for handling the postgresql database migration"""
19+
20+
def __init__(self) -> None:
21+
"""Initialize the Database object with a database connection"""
22+
try:
23+
self.engine = create_engine(os.getenv("DB_URI"), echo=True)
24+
except Exception as e:
25+
print(f"Error creating engine: {e}")
26+
else:
27+
self.connection = self.engine.connect()
28+
29+
def create_database(self):
30+
"""Create the database"""
31+
if not database_exists(self.engine.url):
32+
create_database(self.engine.url)
33+
34+
def run_migrations(self):
35+
"""Run the database migrations"""
36+
try:
37+
self.create_table_pages()
38+
# self.create_table_index()
39+
except Exception as e:
40+
print(f"Error running migrations: {e}")
41+
else:
42+
print("Migrations ran successfully")
43+
44+
def create_table_pages(self):
45+
"""Create the pages table"""
46+
metadata = MetaData()
47+
pages = Table( # noqa: F841
48+
"pages",
49+
metadata,
50+
Column("id", Integer, primary_key=True),
51+
Column("title", String),
52+
Column("md", Text),
53+
Column("url", String),
54+
Column("sub_page", Boolean, default=False),
55+
)
56+
index = Table( # noqa: F841
57+
"index",
58+
metadata,
59+
Column("id", Integer, primary_key=True),
60+
Column("pages_id", ForeignKey("pages.id")),
61+
)
62+
metadata.create_all(self.engine)
63+
self.connection.commit()
64+
65+
def create_table_index(self):
66+
"""Create the index table"""
67+
metadata = MetaData()
68+
index = Table( # noqa: F841
69+
"index",
70+
metadata,
71+
Column("id", Integer, primary_key=True),
72+
Column("pages_id", ForeignKey("pages.id")),
73+
)
74+
metadata.create_all(self.engine)
75+
76+
def drop_tables(self):
77+
"""Drop the tables"""
78+
metadata = MetaData()
79+
pages = Table("pages", metadata) # noqa: F841
80+
index = Table("index", metadata) # noqa: F841
81+
metadata.drop_all(self.engine)
82+
self.connection.commit()
83+
print("Tables dropped successfully")

‎lfweb/database/new_connection.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""Module for handling the database connection"""
2+
3+
import os
4+
from typing import Any
5+
6+
import psycopg2 as psycopg
7+
from loguru import logger
8+
from pydantic import BaseModel, model_validator
9+
10+
11+
class GetDbConnection:
12+
"""Class for handling the database connection as a context manager"""
13+
14+
host = ""
15+
port = ""
16+
username = ""
17+
password = ""
18+
database = ""
19+
connection = None
20+
21+
def __init__(self, database_obj) -> None:
22+
"""Initialize the Database object"""
23+
self.host = database_obj.host
24+
self.port = database_obj.port
25+
self.username = database_obj.username
26+
self.password = database_obj.password
27+
self.database = database_obj.database
28+
29+
def __enter__(self):
30+
logger.info(f"Connecting to database {self.database}")
31+
self.connection = self._get_connection()
32+
return self.connection
33+
34+
def __exit__(self, exc_type, exc_value, traceback):
35+
self.connection.close()
36+
37+
def _get_connection(self, no_db=False):
38+
"""Get a connection to the database"""
39+
return psycopg.connect(
40+
host=self.host,
41+
port=self.port,
42+
user=self.username,
43+
password=self.password,
44+
dbname=self.database if not no_db else None,
45+
)
46+
47+
48+
class DbConnectionCredentials(BaseModel):
49+
"""Class for defining the database connection credentials"""
50+
51+
host: str
52+
port: str
53+
username: str
54+
password: str
55+
database: str
56+
db_uri: str
57+
58+
@model_validator(mode="after")
59+
@classmethod
60+
def set_credentials_from_env(cls, values: Any) -> Any:
61+
"""Set the database connection credentials from the environment"""
62+
values.host = os.getenv("DB_HOST")
63+
values.port = os.getenv("DB_PORT")
64+
values.username = os.getenv("DB_USERNAME")
65+
values.password = os.getenv("DB_PASSWORD")
66+
values.database = os.getenv("DB_NAME")
67+
values.db_uri = os.getenv("DB_URI")
68+
return values

‎lfweb/main/__init__.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
Main routes for the application
33
"""
44

5-
from flask import Blueprint
6-
from loguru import logger
5+
from flask import Blueprint # noqa: F401
6+
from loguru import logger # noqa: F401
77

8-
from .images import bp as images_bp
9-
from .pages_route import bp as pages_bp
10-
from .routes import frontpage_bp
8+
from .images import bp as images_bp # noqa: F401
9+
from .pages_route import bp as pages_bp # noqa: F401
10+
from .routes import frontpage_bp # noqa: F401

‎lfweb/main/routes.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22
Routes for core pages
33
"""
44

5+
import os
6+
57
from flask import Blueprint, render_template
68
from loguru import logger
79

810
from lfweb.members.list import Memberdata
911
from lfweb.pages.index import IndexHandling
12+
from lfweb.utils.doorcount_scrape import DoorCountScraper
1013

1114
frontpage_bp = Blueprint("main", __name__, url_prefix="/", template_folder="templates")
1215

@@ -22,3 +25,33 @@ def frontpage():
2225
memberdata = Memberdata()
2326

2427
return render_template("home.html", pages=index.index, memberdata=memberdata)
28+
29+
30+
@frontpage_bp.route("/memberships")
31+
def memberships():
32+
"""
33+
Renders the memberships page
34+
"""
35+
memberdata = Memberdata()
36+
return render_template("snippets/membership_test.html", memberdata=memberdata)
37+
38+
39+
@frontpage_bp.route("/doorcount")
40+
def doorcount():
41+
"""
42+
Renders the doorcount page
43+
"""
44+
url = os.environ.get("DOORCOUNT_URL")
45+
scraper = DoorCountScraper(url)
46+
page = scraper.fetch_page()
47+
data = scraper.parse_data(page)
48+
return render_template("snippets/doorcount.html", data=data)
49+
50+
51+
@frontpage_bp.route("/membercount")
52+
def membercount():
53+
"""
54+
Renders the membercount page
55+
"""
56+
memberdata = Memberdata()
57+
return render_template("snippets/membercount.html", memberdata=memberdata)

‎lfweb/pages/database.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
"""Module handling page add and remove from the database"""
2+
3+
from loguru import logger
4+
5+
from lfweb.database.connection import DbConnectionCredentials, GetDbConnection
6+
7+
8+
class PagesDatabaseHandling:
9+
"""Class for handling pages insertion and deletion in the databases pages and the index table"""
10+
11+
def __init__(self):
12+
try:
13+
self.database_obj = DbConnectionCredentials()
14+
except Exception as e:
15+
logger.error(f"Error getting database credentials from env. vars: {e}")
16+
17+
def add_page_to_index(self, title: str, url: str, md: str, sub_page: bool) -> bool:
18+
"""Method adding a page to the index table"""
19+
with GetDbConnection(self.database_obj) as conn:
20+
with conn.cursor() as cur:
21+
cur.execute(
22+
"""
23+
INSERT INTO pages (title, url, md)
24+
VALUES (%(title)s, %(url)s ,%(md)s)
25+
""",
26+
{"title": title, "url": url, "md": md},
27+
)
28+
conn.commit()
29+
30+
return conn.close()

0 commit comments

Comments
 (0)