Skip to content

Commit c68cd70

Browse files
committed
feat: Add analytics endpoints with Polars and OpenTelemetry - comprehensive analytics functionality with API endpoints, data processing, observability, and tests
1 parent 6c9b1fa commit c68cd70

File tree

7 files changed

+311
-1
lines changed

7 files changed

+311
-1
lines changed

README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,20 @@
2424
- 🚢 Deployment instructions using Docker Compose, including how to set up a frontend Traefik proxy to handle automatic HTTPS certificates.
2525
- 🏭 CI (continuous integration) and CD (continuous deployment) based on GitHub Actions.
2626

27+
## Analytics and Observability
28+
29+
This project now includes enhanced analytics capabilities and observability through OpenTelemetry:
30+
31+
* **New Analytics Endpoints**:
32+
* `/api/v1/analytics/user-summary`: Provides a summary of user statistics, including total users, active/inactive counts, and signup trends (if user creation timestamps are available).
33+
* `/api/v1/analytics/item-trends`: Provides a summary of item statistics, including total items and creation trends (if item creation timestamps are available).
34+
These endpoints utilize Polars for efficient in-memory data aggregation.
35+
36+
* **OpenTelemetry Integration**:
37+
* The backend is instrumented with OpenTelemetry for distributed tracing. This provides insights into request flows and database interactions.
38+
* To export traces, configure the OTLP exporter endpoint via the environment variable: `OTEL_EXPORTER_OTLP_ENDPOINT="<your_otlp_collector_url:port>"` (e.g., `http://localhost:4317`).
39+
* You can also customize the service name reported to your observability platform using the `OTEL_SERVICE_NAME` environment variable.
40+
2741
### Dashboard Login
2842

2943
[![API docs](img/login.png)](https://github.com/fastapi/full-stack-fastapi-template)

backend/app/api/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
from fastapi import APIRouter
22

3-
from app.api.routes import items, login, private, users, utils
3+
from app.api.routes import items, login, private, users, utils, analytics
44
from app.core.config import settings
55

66
api_router = APIRouter()
77
api_router.include_router(login.router)
88
api_router.include_router(users.router)
99
api_router.include_router(utils.router)
1010
api_router.include_router(items.router)
11+
api_router.include_router(analytics.router)
1112

1213

1314
if settings.ENVIRONMENT == "local":
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
from fastapi import APIRouter, Depends, HTTPException
2+
from sqlmodel import Session, select
3+
import polars as pl
4+
from typing import List, Dict, Any # Dict and Any might be needed for Polars conversion
5+
from datetime import date
6+
import pydantic # Ensure pydantic is imported
7+
8+
from app.models import User, Item # Assuming User model is in app.models, Import Item
9+
from app.api.deps import SessionDep, get_current_active_superuser # SessionDep for dependency injection
10+
11+
from opentelemetry import trace
12+
13+
# Pydantic models for response
14+
class UserSignupTrend(pydantic.BaseModel): # Corrected: pydantic.BaseModel
15+
signup_date: date
16+
count: int
17+
18+
class UserActivity(pydantic.BaseModel): # Corrected: pydantic.BaseModel
19+
active_users: int
20+
inactive_users: int
21+
22+
class UserAnalyticsSummary(pydantic.BaseModel): # Corrected: pydantic.BaseModel
23+
total_users: int
24+
signup_trends: List[UserSignupTrend]
25+
activity_summary: UserActivity
26+
# Add more fields as desired, e.g., average_items_per_user: float
27+
28+
# Pydantic models for Item analytics
29+
class ItemCreationTrend(pydantic.BaseModel):
30+
creation_date: date
31+
count: int
32+
33+
# class ItemOwnerDistribution(pydantic.BaseModel): # Optional for now
34+
# owner_id: str
35+
# item_count: int
36+
37+
class ItemAnalyticsTrends(pydantic.BaseModel):
38+
total_items: int
39+
creation_trends: List[ItemCreationTrend]
40+
# owner_distribution: List[ItemOwnerDistribution] # Optional
41+
42+
43+
router = APIRouter(prefix="/analytics", tags=["analytics"])
44+
tracer = trace.get_tracer(__name__)
45+
46+
@router.get("/user-summary", response_model=UserAnalyticsSummary)
47+
def get_user_summary(session: SessionDep): # get_current_active_superuser is imported but not used here yet
48+
with tracer.start_as_current_span("user_summary_endpoint"):
49+
50+
users_list: List[User]
51+
with tracer.start_as_current_span("fetch_all_users_sql"):
52+
statement = select(User)
53+
users_list = session.exec(statement).all()
54+
55+
if not users_list:
56+
return UserAnalyticsSummary(
57+
total_users=0,
58+
signup_trends=[],
59+
activity_summary=UserActivity(active_users=0, inactive_users=0)
60+
)
61+
62+
with tracer.start_as_current_span("convert_users_to_polars"):
63+
users_data = []
64+
for user in users_list:
65+
user_dict = {
66+
"id": user.id, # No explicit str() casting for now, per instructions
67+
"email": user.email,
68+
"is_active": user.is_active,
69+
"is_superuser": user.is_superuser,
70+
"full_name": user.full_name,
71+
}
72+
# Attempt to get 'created_at' if it exists (it doesn't in the standard model)
73+
if hasattr(user, 'created_at') and user.created_at:
74+
user_dict['created_at'] = user.created_at
75+
users_data.append(user_dict)
76+
77+
if not users_data: # Should not happen if users_list is not empty, but as a safe guard
78+
return UserAnalyticsSummary(
79+
total_users=0,
80+
signup_trends=[],
81+
activity_summary=UserActivity(active_users=0, inactive_users=0)
82+
)
83+
84+
# Create DataFrame without explicit casting of 'id' first.
85+
# If Polars errors on UUID, the instruction is to add:
86+
# df_users = df_users.with_columns(pl.col('id').cast(pl.Utf8))
87+
df_users = pl.DataFrame(users_data)
88+
89+
90+
total_users = df_users.height # More idiomatic for Polars than len(df_users)
91+
92+
with tracer.start_as_current_span("calculate_user_activity_polars"):
93+
active_users = df_users.filter(pl.col("is_active") == True).height
94+
inactive_users = total_users - active_users
95+
96+
signup_trends_data = []
97+
if 'created_at' in df_users.columns: # This will be false as User model has no created_at
98+
with tracer.start_as_current_span("calculate_signup_trends_polars"):
99+
# Ensure 'created_at' is a datetime type. If string, parse it.
100+
# Assuming it's already a datetime.date or datetime.datetime from SQLModel
101+
# If it's datetime, cast to date for daily trends
102+
df_users_with_date = df_users.with_columns(pl.col("created_at").cast(pl.Date).alias("signup_day"))
103+
104+
signup_counts_df = df_users_with_date.group_by("signup_day").agg(
105+
pl.count().alias("count")
106+
).sort("signup_day")
107+
108+
signup_trends_data = [
109+
UserSignupTrend(signup_date=row["signup_day"], count=row["count"])
110+
for row in signup_counts_df.to_dicts()
111+
]
112+
113+
return UserAnalyticsSummary(
114+
total_users=total_users,
115+
signup_trends=signup_trends_data, # Will be empty as 'created_at' is not in User model
116+
activity_summary=UserActivity(active_users=active_users, inactive_users=inactive_users)
117+
)
118+
119+
@router.get("/item-trends", response_model=ItemAnalyticsTrends)
120+
def get_item_trends(session: SessionDep):
121+
with tracer.start_as_current_span("item_trends_endpoint"):
122+
123+
items_list: List[Item]
124+
with tracer.start_as_current_span("fetch_all_items_sql"):
125+
statement = select(Item)
126+
items_list = session.exec(statement).all()
127+
128+
if not items_list:
129+
return ItemAnalyticsTrends(
130+
total_items=0,
131+
creation_trends=[]
132+
# owner_distribution=[] # Optional
133+
)
134+
135+
with tracer.start_as_current_span("convert_items_to_polars"):
136+
items_data = []
137+
for item in items_list:
138+
item_dict = {
139+
"id": str(item.id), # Cast UUID to string
140+
"title": item.title,
141+
"description": item.description,
142+
"owner_id": str(item.owner_id) # Cast UUID to string
143+
}
144+
# IMPORTANT: Item model does not have 'created_at'.
145+
# This will result in empty creation_trends.
146+
if hasattr(item, 'created_at') and item.created_at:
147+
item_dict['created_at'] = item.created_at
148+
items_data.append(item_dict)
149+
150+
if not items_data: # Safety check
151+
return ItemAnalyticsTrends(total_items=0, creation_trends=[])
152+
153+
df_items = pl.DataFrame(items_data)
154+
155+
total_items = df_items.height
156+
157+
creation_trends_data = []
158+
if 'created_at' in df_items.columns:
159+
with tracer.start_as_current_span("calculate_item_creation_trends_polars"):
160+
# Ensure 'created_at' is datetime, then cast to date for daily trends
161+
df_items_with_date = df_items.with_columns(
162+
pl.col("created_at").cast(pl.Date).alias("creation_day")
163+
)
164+
165+
creation_counts_df = df_items_with_date.group_by("creation_day").agg(
166+
pl.count().alias("count")
167+
).sort("creation_day")
168+
169+
creation_trends_data = [
170+
ItemCreationTrend(creation_date=row["creation_day"], count=row["count"])
171+
for row in creation_counts_df.to_dicts()
172+
]
173+
174+
# Placeholder for owner distribution if implemented later
175+
# owner_distribution_data = []
176+
177+
return ItemAnalyticsTrends(
178+
total_items=total_items,
179+
creation_trends=creation_trends_data
180+
# owner_distribution=owner_distribution_data # Optional
181+
)

backend/app/core/telemetry.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import os
2+
from opentelemetry import trace
3+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
4+
from opentelemetry.sdk.resources import Resource
5+
from opentelemetry.sdk.trace import TracerProvider
6+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
7+
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
8+
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
9+
10+
from app.core.db import engine # Assuming engine is in app.core.db
11+
from app.core.config import settings # To potentially get service name
12+
13+
def init_telemetry(app):
14+
# Set service name, try from settings or default
15+
service_name = getattr(settings, "OTEL_SERVICE_NAME", "fastapi-application")
16+
17+
resource = Resource(attributes={
18+
"service.name": service_name
19+
})
20+
21+
# Configure OTLP exporter
22+
# Endpoint can be configured via OTEL_EXPORTER_OTLP_ENDPOINT environment variable
23+
# Defaulting to http://localhost:4317 if not set, as per OpenTelemetry spec
24+
otlp_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
25+
26+
span_exporter = OTLPSpanExporter(
27+
endpoint=otlp_endpoint,
28+
# insecure=True # Set to True if your collector is not using TLS. Adjust as needed.
29+
)
30+
31+
processor = BatchSpanProcessor(span_exporter)
32+
33+
provider = TracerProvider(resource=resource)
34+
provider.add_span_processor(processor)
35+
36+
# Sets the global default tracer provider
37+
trace.set_tracer_provider(provider)
38+
39+
# Instrument FastAPI
40+
FastAPIInstrumentor.instrument_app(app)
41+
42+
# Instrument SQLAlchemy
43+
# Ensure the engine is already configured/available when this is called.
44+
SQLAlchemyInstrumentor().instrument(engine=engine)
45+
46+
# You can get a tracer instance and create spans if needed for custom instrumentation later
47+
# tracer = trace.get_tracer(__name__)

backend/app/main.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
from app.api.main import api_router
77
from app.core.config import settings
8+
from app.core.telemetry import init_telemetry # Import the new function
89

910

1011
def custom_generate_unique_id(route: APIRoute) -> str:
@@ -20,6 +21,9 @@ def custom_generate_unique_id(route: APIRoute) -> str:
2021
generate_unique_id_function=custom_generate_unique_id,
2122
)
2223

24+
# Initialize OpenTelemetry
25+
init_telemetry(app)
26+
2327
# Set all CORS enabled origins
2428
if settings.all_cors_origins:
2529
app.add_middleware(
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
from fastapi.testclient import TestClient
2+
from sqlmodel import Session # For potential future use with fixtures
3+
import pytest # For potential future use with fixtures
4+
5+
from app.core.config import settings
6+
# Assuming your FastAPI app instance is named 'app' in 'app.main'
7+
# Adjust the import if your app instance is located elsewhere for tests
8+
# from app.main import app
9+
10+
# Expected Pydantic response models (adjust import path if they are moved)
11+
# from app.api.routes.analytics import UserAnalyticsSummary, ItemAnalyticsTrends
12+
13+
# Test for User Analytics Summary
14+
def test_get_user_summary(client: TestClient) -> None:
15+
response = client.get(f"{settings.API_V1_STR}/analytics/user-summary")
16+
assert response.status_code == 200
17+
data = response.json()
18+
19+
assert "total_users" in data
20+
assert isinstance(data["total_users"], int)
21+
22+
assert "signup_trends" in data
23+
assert isinstance(data["signup_trends"], list)
24+
25+
assert "activity_summary" in data
26+
assert "active_users" in data["activity_summary"]
27+
assert "inactive_users" in data["activity_summary"]
28+
assert isinstance(data["activity_summary"]["active_users"], int)
29+
assert isinstance(data["activity_summary"]["inactive_users"], int)
30+
31+
# Check if signup_trends items have the correct structure if not empty
32+
if data["signup_trends"]:
33+
trend_item = data["signup_trends"][0]
34+
assert "signup_date" in trend_item
35+
assert "count" in trend_item
36+
assert isinstance(trend_item["count"], int)
37+
38+
39+
# Test for Item Analytics Trends
40+
def test_get_item_trends(client: TestClient) -> None:
41+
response = client.get(f"{settings.API_V1_STR}/analytics/item-trends")
42+
assert response.status_code == 200
43+
data = response.json()
44+
45+
assert "total_items" in data
46+
assert isinstance(data["total_items"], int)
47+
48+
assert "creation_trends" in data
49+
assert isinstance(data["creation_trends"], list)
50+
51+
# Check if creation_trends items have the correct structure if not empty
52+
if data["creation_trends"]:
53+
trend_item = data["creation_trends"][0]
54+
assert "creation_date" in trend_item
55+
assert "count" in trend_item
56+
assert isinstance(trend_item["count"], int)

backend/pyproject.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ dependencies = [
2121
"pydantic-settings<3.0.0,>=2.2.1",
2222
"sentry-sdk[fastapi]<2.0.0,>=1.40.6",
2323
"pyjwt<3.0.0,>=2.8.0",
24+
"duckdb",
25+
"polars",
26+
"opentelemetry-api",
27+
"opentelemetry-sdk",
28+
"opentelemetry-exporter-otlp",
29+
"opentelemetry-instrumentation-fastapi",
30+
"opentelemetry-instrumentation-sqlalchemy",
2431
]
2532

2633
[tool.uv]

0 commit comments

Comments
 (0)