feat: utm param support

explodingcamera · explodingcamera · commit 4a1e5d8cb68c · 2024-11-04T01:01:56.000+01:00
Signed-off-by: Henry Gressmann &lt;mail@henrygressmann.de&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,11 +16,22 @@ The format is roughly based on the output of `git-cliff` and this project adhere
 Since this is not a library, this changelog focuses on the changes that are relevant to the end-users. For a detailed list of changes, see the commit history, which adheres to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). New releases are created automatically when a new tag is pushed (Commit message: chore(release): vX.X.X).
 -->
 
-<!-- ## [Unreleased] -->
+## [Unreleased]
+
+### 🚀 Features
+
+- **UTM parameters**: Added support for UTM parameters. You can filter and search by UTM source, medium, campaign, content, and term.
+
+### 🐛 Bug Fixes
+
+- Fixed a potential panic when entities are not found in the database (https://github.com/explodingcamera/liwan/commit/31405a721dc5c5493098e211927281cca7816fec)
+- Fixed issues with the `Yesterday` Date Range (https://github.com/explodingcamera/liwan/commit/76278b579c5fe1557bf1c184542ed6ed2aba57cd)
 
 ## **Liwan v0.1.1** - 2024-09-24
 
-This release just drops some of the duckdb indexes to improve performance and reduce the database size.
+### ⚡ Performance
+
+- **Database indexes**: Removed all unnecessary indexes to improve performance and reduce disk usage (https://github.com/explodingcamera/liwan/commit/6191a72f08e8659237bc6c22139bde94432f66bb)
 
 ## **Liwan v0.1.0** - 2024-09-18
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 name="liwan"
 version="0.1.1"
 edition="2021"
-rust-version="1.80"
+rust-version="1.82"
 repository="https://github.com/explodingcamera/liwan"
 license="AGPL-3.0-only OR LicenseRef-Liwan"
 publish=false
@@ -48,7 +48,6 @@ tracing-subscriber={version="0.3", features=["env-filter"]}
 poem={version="3.1", default-features=false, features=[
     "time",
     "embed",
-    "session",
     "cookie",
     "compression",
     "tower-compat",
diff --git a/src/app/core/entities.rs b/src/app/core/entities.rs
@@ -1,9 +1,7 @@
 use eyre::{bail, Result};
 
-use crate::{
-    app::{models, SqlitePool},
-    utils::validate,
-};
+use crate::app::{models, SqlitePool};
+use crate::utils::validate;
 
 #[derive(Clone)]
 pub struct LiwanEntities {
diff --git a/src/app/core/events.rs b/src/app/core/events.rs
@@ -4,13 +4,9 @@ use crossbeam::{channel::Receiver, sync::ShardedLock};
 use eyre::{bail, Result};
 use time::OffsetDateTime;
 
-use crate::{
-    app::{
-        models::{event_params, Event},
-        DuckDBPool, SqlitePool, EVENT_BATCH_INTERVAL,
-    },
-    utils::hash::generate_salt,
-};
+use crate::app::models::{event_params, Event};
+use crate::app::{DuckDBPool, SqlitePool, EVENT_BATCH_INTERVAL};
+use crate::utils::hash::generate_salt;
 
 #[derive(Clone)]
 pub struct LiwanEvents {
diff --git a/src/app/core/geoip.rs b/src/app/core/geoip.rs
@@ -1,14 +1,10 @@
 #![allow(dead_code)]
 
-use std::{
-    collections::HashMap,
-    net::IpAddr,
-    path::{Path, PathBuf},
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-};
+use std::collections::HashMap;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
 
 use crate::app::SqlitePool;
 use crossbeam::sync::ShardedLock;
diff --git a/src/app/core/projects.rs b/src/app/core/projects.rs
@@ -1,9 +1,7 @@
 use eyre::{bail, Result};
 
-use crate::{
-    app::{models, SqlitePool},
-    utils::validate,
-};
+use crate::app::{models, SqlitePool};
+use crate::utils::validate;
 
 #[derive(Clone)]
 pub struct LiwanProjects {
diff --git a/src/app/core/reports.rs b/src/app/core/reports.rs
@@ -56,6 +56,11 @@ pub enum Dimension {
     Mobile,
     Country,
     City,
+    UtmSource,
+    UtmMedium,
+    UtmCampaign,
+    UtmContent,
+    UtmTerm,
 }
 
 #[derive(Enum, Debug, Clone, Copy, PartialEq)]
@@ -174,6 +179,11 @@ fn filter_sql(filters: &[DimensionFilter]) -> Result<(String, ParamVec)> {
                 Dimension::Mobile => format!("mobile::text {filter_value}"),
                 Dimension::Country => format!("country {filter_value}"),
                 Dimension::City => format!("city {filter_value}"),
+                Dimension::UtmSource => format!("utm_source {filter_value}"),
+                Dimension::UtmMedium => format!("utm_medium {filter_value}"),
+                Dimension::UtmCampaign => format!("utm_campaign {filter_value}"),
+                Dimension::UtmContent => format!("utm_content {filter_value}"),
+                Dimension::UtmTerm => format!("utm_term {filter_value}"),
             })
         })
         .collect::<Result<Vec<String>>>()?;
@@ -401,6 +411,11 @@ pub fn dimension_report(
         Dimension::Mobile => ("mobile::text", "mobile"),
         Dimension::Country => ("country", "country"),
         Dimension::City => ("concat(country, city)", "country, city"),
+        Dimension::UtmSource => ("utm_source", "utm_source"),
+        Dimension::UtmMedium => ("utm_medium", "utm_medium"),
+        Dimension::UtmCampaign => ("utm_campaign", "utm_campaign"),
+        Dimension::UtmContent => ("utm_content", "utm_content"),
+        Dimension::UtmTerm => ("utm_term", "utm_term"),
     };
 
     params.push(range.start());
diff --git a/src/app/core/users.rs b/src/app/core/users.rs
@@ -1,10 +1,6 @@
-use crate::{
-    app::{models, SqlitePool},
-    utils::{
-        hash::{hash_password, verify_password},
-        validate,
-    },
-};
+use crate::app::{models, SqlitePool};
+use crate::utils::hash::{hash_password, verify_password};
+use crate::utils::validate;
 use eyre::{bail, Result};
 
 #[derive(Clone)]
diff --git a/src/app/db.rs b/src/app/db.rs
@@ -3,7 +3,7 @@ use crate::utils::refinery_duckdb::DuckDBConnection;
 use crate::utils::refinery_sqlite::RqlConnection;
 
 use duckdb::DuckdbConnectionManager;
-use eyre::Result;
+use eyre::{bail, Result};
 use r2d2_sqlite::SqliteConnectionManager;
 use refinery::Runner;
 use std::path::PathBuf;
@@ -16,7 +16,17 @@ pub(super) fn init_duckdb(
     let conn = DuckdbConnectionManager::file(path)?;
     let pool = r2d2::Pool::new(conn)?;
     migrations_runner.set_migration_table_name("migrations");
-    migrations_runner.run(&mut DuckDBConnection(pool.get()?))?;
+
+    for migration in migrations_runner.run_iter(&mut DuckDBConnection(pool.get()?)) {
+        match migration {
+            Ok(migration) => {
+                tracing::info!("Applied migration: {}", migration);
+            }
+            Err(err) => {
+                bail!("Failed to apply migration: {}", err);
+            }
+        }
+    }
 
     {
         let conn = pool.get()?;
diff --git a/src/app/models.rs b/src/app/models.rs
@@ -12,6 +12,11 @@ pub struct Event {
     pub mobile: Option<bool>,
     pub country: Option<String>,
     pub city: Option<String>,
+    pub utm_source: Option<String>,
+    pub utm_medium: Option<String>,
+    pub utm_campaign: Option<String>,
+    pub utm_content: Option<String>,
+    pub utm_term: Option<String>,
 }
 
 #[derive(Debug, Clone)]
@@ -83,6 +88,11 @@ macro_rules! event_params {
             $event.mobile,
             $event.country,
             $event.city,
+            $event.utm_source,
+            $event.utm_medium,
+            $event.utm_campaign,
+            $event.utm_content,
+            $event.utm_term
         ]
     };
 }
diff --git a/src/migrations/events/V2__remove_indexes.sql b/src/migrations/events/V2__remove_indexes.sql
@@ -0,0 +1,6 @@
+drop index events_event_idx;
+drop index events_entity_id_idx;
+drop index events_visitor_id_idx;
+drop index events_created_at_idx;
+drop index events_entity_id_created_at_idx;
+drop index events_visitor_id_created_at_idx;
diff --git a/src/migrations/events/V3__utm_params.sql b/src/migrations/events/V3__utm_params.sql
@@ -0,0 +1,5 @@
+alter table events add column utm_source text;
+alter table events add column utm_medium text;
+alter table events add column utm_campaign text;
+alter table events add column utm_content text;
+alter table events add column utm_term text;
diff --git a/src/migrations/events/v2__remove_indexes.sql b/src/migrations/events/v2__remove_indexes.sql
diff --git a/src/utils/seed.rs b/src/utils/seed.rs
@@ -19,6 +19,11 @@ const CITIES: &[(&str, &str)] = &[
     ("Tokyo", "JP"),
     ("Sydney", "AU"),
 ];
+const UTM_CAMPAIGNS: &[&str] = &["", "summer_sale", "black_friday", "christmas", "new_year"];
+const UTM_CONTENTS: &[&str] = &["", "banner", "sidebar", "footer", "popup"];
+const UTM_MEDIUMS: &[&str] = &["", "cpc", "organic", "referral", "email"];
+const UTM_SOURCES: &[&str] = &["", "google", "bing", "facebook", "twitter"];
+const UTM_TERMS: &[&str] = &["", "liwan", "analytics", "tracking", "web"];
 
 pub fn random_events(
     time_range: (OffsetDateTime, OffsetDateTime),
@@ -59,6 +64,11 @@ pub fn random_events(
             referrer: if referrer.is_empty() { None } else { Some(referrer.to_string()) },
             path: Some(path.to_string()),
             visitor_id: random_el(&visitor_ids, 0.7).to_string(),
+            utm_campaign: Some(random_el(UTM_CAMPAIGNS, 0.5).to_string()),
+            utm_content: Some(random_el(UTM_CONTENTS, 0.5).to_string()),
+            utm_medium: Some(random_el(UTM_MEDIUMS, 0.5).to_string()),
+            utm_source: Some(random_el(UTM_SOURCES, 0.5).to_string()),
+            utm_term: Some(random_el(UTM_TERMS, 0.5).to_string()),
         })
     })
 }
diff --git a/src/web/routes/event.rs b/src/web/routes/event.rs
@@ -21,6 +21,16 @@ struct EventRequest {
     name: String,
     url: String,
     referrer: Option<String>,
+    utm: Option<Utm>,
+}
+
+#[derive(Object)]
+struct Utm {
+    source: Option<String>,
+    content: Option<String>,
+    medium: Option<String>,
+    campaign: Option<String>,
+    term: Option<String>,
 }
 
 thread_local! {
@@ -94,6 +104,11 @@ impl EventApi {
             path: path.into(),
             mobile: Some(useragent::is_mobile(&client)),
             platform: client.os.family.to_string().into(),
+            utm_campaign: event.utm.as_ref().and_then(|u| u.campaign.clone()),
+            utm_content: event.utm.as_ref().and_then(|u| u.content.clone()),
+            utm_medium: event.utm.as_ref().and_then(|u| u.medium.clone()),
+            utm_source: event.utm.as_ref().and_then(|u| u.source.clone()),
+            utm_term: event.utm.as_ref().and_then(|u| u.term.clone()),
         };
 
         let _ = events.try_send(event);
diff --git a/tracker/package.json b/tracker/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "liwan-tracker",
-	"version": "0.1.0-beta.2",
+	"version": "0.2.0",
 	"homepage": "https://liwan.dev",
 	"repository": {
 		"type": "git",
diff --git a/tracker/script.min.js b/tracker/script.min.js
diff --git a/tracker/script.ts b/tracker/script.ts
@@ -10,6 +10,7 @@ type Payload = {
 	name: string;
 	url: string;
 	referrer?: string;
+	utm?: { campaign?: string; content?: string; medium?: string; source?: string; term?: string };
 };
 
 export type EventOptions = {
@@ -86,6 +87,7 @@ export async function event(name = "pageview", options?: EventOptions) {
 	if (typeof localStorage !== "undefined" && localStorage.getItem("disable-liwan")) return ignore("localStorage flag");
 	if (LOCALHOST_REGEX.test(location.hostname) || location.protocol === "file:") return ignore("localhost");
 	if (!endpoint && !options?.endpoint) return ignore("no endpoint");
+	const utm = new URLSearchParams(location.search);
 
 	// biome-ignore lint/style/noNonNullAssertion: we know that endpoint is not null
 	return fetch((options?.endpoint || endpoint)!, {
@@ -96,6 +98,13 @@ export async function event(name = "pageview", options?: EventOptions) {
 			name,
 			referrer: options?.referrer || referrer,
 			url: options?.url || `${location.origin}${location.pathname}`,
+			utm: {
+				campaign: utm.get("utm_campaign"),
+				content: utm.get("utm_content"),
+				medium: utm.get("utm_medium"),
+				source: utm.get("utm_source"),
+				term: utm.get("utm_term"),
+			},
 		}),
 	}).then((response) => {
 		if (!response.ok) log(`Failed to send event: ${response.statusText}`);
diff --git a/web/src/api/constants.ts b/web/src/api/constants.ts
@@ -17,6 +17,11 @@ export const dimensionNames: Record<Dimension, string> = {
 	city: "City",
 	country: "Country",
 	fqdn: "Domain",
+	utm_campaign: "UTM Campaign",
+	utm_content: "UTM Content",
+	utm_medium: "UTM Medium",
+	utm_source: "UTM Source",
+	utm_term: "UTM Term",
 };
 
 export const filterNames: Record<DimensionFilter["filterType"], string> = {
diff --git a/web/src/api/dashboard.ts b/web/src/api/dashboard.ts
diff --git a/web/src/api/types.ts b/web/src/api/types.ts
