chore: add more tests & improve api performance

Signed-off-by: Henry Gressmann <[email protected]>

Author: explodingcamera

CHANGELOG.md

@@ -27,13 +27,14 @@ Since this is not a library, this changelog focuses on the changes that are rele
 
 ### 🐛 Bug Fixes
 
-- Added more tests that fixed some smaller bugs
 - Fixed a potential panic when entities are not found in the database ([`31405a72`](https://github.com/explodingcamera/liwan/commit/31405a721dc5c5493098e211927281cca7816fec))
 - Fixed issues with the `Yesterday` Date Range ([`76278b57`](https://github.com/explodingcamera/liwan/commit/76278b579c5fe1557bf1c184542ed6ed2aba57cd))
 
 ### Other
 
 - Removed Sessions and Average Views per Session metrics. They were not accurate and were removed to avoid confusion.
+- Added more tests & improved API performance
+- Updated dependencies
 
 ## **Liwan v0.1.1** - 2024-09-24
 

Cargo.lock

@@ -85,6 +85,7 @@ cookie={version="*"}
 [features]
 default=["geoip"]
 geoip=["dep:maxminddb"]
+_enable_seeding=[]
 
 [profile.dev]
 opt-level=1
@@ -93,3 +94,8 @@ incremental=true
 [profile.release]
 lto="thin"
 strip=true
+
+[profile.dev.package.duckdb]
+inherits="release"
+[profile.dev.package.rusqlite]
+inherits="release"

Cargo.toml

@@ -1,8 +1,8 @@
 <br/>
 
 <div align="center">
-    <a href=""><img  src="./web/public/favicon.svg" width="100px"></a>
     <h2>
+        <img float="left" src="./web/public/favicon.svg" width="16px"/>
         <a href="https://liwan.dev">liwan.dev</a> - Easy & Privacy-First Web Analytics
     </h2>
     <div>

README.md

@@ -7,7 +7,7 @@ use duckdb::params_from_iter;
 use eyre::{bail, Result};
 use poem_openapi::{Enum, Object};
 
-#[derive(Object)]
+#[derive(Object, Debug, Clone)]
 pub struct DateRange {
     pub start: time::OffsetDateTime,
     pub end: time::OffsetDateTime,
@@ -83,7 +83,7 @@ pub struct ReportStats {
     pub avg_time_on_site: f64,
 }
 
-#[derive(Object, Debug)]
+#[derive(Object, Debug, Clone)]
 #[oai(rename_all = "camelCase")]
 pub struct DimensionFilter {
     /// The dimension to filter by
@@ -195,17 +195,9 @@ fn metric_sql(metric: Metric) -> String {
             // bounce sessions: time to next / time to prev are both null or both > 1800
             "--sql
             count(distinct sd.visitor_id)
-                filter (
-                    where
-                        (sd.time_until_next_event is null or sd.time_until_next_event > 1800) and
-                        (sd.time_since_previous_event is null or sd.time_since_previous_event > 1800)
-                )
-            /
-            count(distinct sd.visitor_id)
-                filter (
-                    where
-                        sd.time_until_next_event is null or sd.time_until_next_event > 1800
-                )
+                filter (where (sd.time_until_next_event is null or sd.time_until_next_event > 1800) and
+                             (sd.time_since_previous_event is null or sd.time_since_previous_event > 1800)) /
+            count(distinct sd.visitor_id) filter (where sd.time_until_next_event is null or sd.time_until_next_event > 1800)
             "
         }
         Metric::AvgTimeOnSite => {
@@ -311,8 +303,7 @@ pub fn overall_report(
                 from events, params
                 where
                     event = ?::text and
-                    created_at >= params.start_time and
-                    created_at <= params.end_time and
+                    created_at between params.start_time and params.end_time and
                     entity_id in ({entity_vars})
                     {filters_sql}
             ),
@@ -365,8 +356,6 @@ pub fn overall_stats(
         return Ok(ReportStats::default());
     }
 
-    let mut params = ParamVec::new();
-
     let entity_vars = repeat_vars(entities.len());
     let (filters_sql, filters_params) = filter_sql(filters)?;
 
@@ -375,6 +364,7 @@ pub fn overall_stats(
     let metric_bounce_rate = metric_sql(Metric::BounceRate);
     let metric_avg_time_on_site = metric_sql(Metric::AvgTimeOnSite);
 
+    let mut params = ParamVec::new();
     params.push(range.start);
     params.push(range.end);
     params.push(event);
@@ -399,11 +389,10 @@ pub fn overall_stats(
                 from events, params
                 where
                     event = ?::text and
-                    created_at >= params.start_time and
-                    created_at <= params.end_time and
+                    created_at between params.start_time and params.end_time and
                     entity_id in ({entity_vars})
                     {filters_sql}
-            ) 
+            )
         select
             {metric_total} as total_views,
             {metric_unique_visitors} as unique_visitors,
@@ -485,10 +474,9 @@ pub fn dimension_report(
                     extract(epoch from (created_at - lag(created_at) over (partition by visitor_id order by created_at))) as time_since_previous_event
                 from events sd, params
                 where
-                    sd.event = ?::text and
-                    sd.created_at >= params.start_time and
-                    sd.created_at <= params.end_time and
-                    sd.entity_id in ({entity_vars})
+                    event = ?::text and
+                    created_at between params.start_time and params.end_time and
+                    entity_id in ({entity_vars})
                     {filters_sql}
                 group by
                     {group_by_columns}, visitor_id, created_at

src/app/core/reports.rs

@@ -1,5 +1,6 @@
-use crate::app::SqlitePool;
+use crate::app::{models, SqlitePool};
 use eyre::Result;
+use rusqlite::params;
 use time::OffsetDateTime;
 
 #[derive(Clone)]
@@ -20,17 +21,40 @@ impl LiwanSessions {
         Ok(())
     }
 
-    /// Get the username associated with a session ID, if the session is still valid.
+    /// Get the user associated with a session ID, if the session is still valid.
     /// Returns None if the session is expired
-    pub fn get(&self, session_id: &str) -> Result<Option<String>> {
+    pub fn get(&self, session_id: &str) -> Result<Option<models::User>> {
         let conn = self.pool.get()?;
-        let mut stmt = conn.prepare_cached("select username, expires_at from sessions where id = ?")?;
-        let (username, expires_at): (String, OffsetDateTime) =
-            stmt.query_row([session_id], |row| Ok((row.get("username")?, row.get("expires_at")?)))?;
-        if expires_at < OffsetDateTime::now_utc() {
-            return Ok(None);
-        }
-        Ok(Some(username))
+
+        let mut stmt = conn.prepare_cached(
+            r#"--sql
+            select u.username, u.role, u.projects
+            from sessions s
+            join users u
+            on lower(u.username) = lower(s.username)
+            where
+                s.id = ?
+                and s.expires_at > ?
+        "#,
+        )?;
+
+        let user = stmt.query_row(params![session_id, time::OffsetDateTime::now_utc()], |row| {
+            Ok(models::User {
+                username: row.get("username")?,
+                role: row.get::<_, String>("role")?.try_into().unwrap_or_default(),
+                projects: row.get::<_, String>("projects")?.split(',').map(str::to_string).collect(),
+            })
+        });
+
+        user.map(Some).or_else(
+            |err| {
+                if err == rusqlite::Error::QueryReturnedNoRows {
+                    Ok(None)
+                } else {
+                    Err(err.into())
+                }
+            },
+        )
     }
 
     /// Delete a session

src/app/core/sessions.rs

@@ -102,11 +102,10 @@ impl Liwan {
     }
 }
 
-#[cfg(debug_assertions)]
+#[cfg(any(debug_assertions, test, feature = "_enable_seeding"))]
 impl Liwan {
-    pub fn seed_database(&self) -> Result<()> {
+    pub fn seed_database(&self, count_per_entity: usize) -> Result<()> {
         use models::UserRole;
-        use rand::Rng;
         use time::OffsetDateTime;
 
         let entities = vec![
@@ -140,13 +139,10 @@ impl Liwan {
                 &models::Entity { id: entity_id.to_string(), display_name: display_name.to_string() },
                 &project_ids,
             )?;
-            let events = crate::utils::seed::random_events(
-                (start, end),
-                entity_id,
-                fqdn,
-                rand::thread_rng().gen_range(5000..20000),
-            );
+            let events = crate::utils::seed::random_events((start, end), entity_id, fqdn, count_per_entity);
+            let now = std::time::Instant::now();
             self.events.append(events)?;
+            tracing::info!("Seeded entity {} in {:?}", entity_id, now.elapsed());
         }
 
         Ok(())

src/app/mod.rs

@@ -32,11 +32,11 @@ pub enum Command {
     UpdatePassword(UpdatePassword),
     AddUser(AddUser),
     Users(ListUsers),
-    #[cfg(debug_assertions)]
+    #[cfg(any(debug_assertions, test, feature = "_enable_seeding"))]
     SeedDatabase(SeedDatabase),
 }
 
-#[cfg(debug_assertions)]
+#[cfg(any(debug_assertions, test, feature = "_enable_seeding"))]
 #[derive(FromArgs)]
 #[argh(subcommand, name = "seed-database")]
 /// Seed the database with some test data
@@ -130,10 +130,10 @@ pub fn handle_command(mut config: Config, cmd: Command) -> Result<()> {
             std::fs::write(&output, DEFAULT_CONFIG)?;
             println!("Configuration file written to liwan.config.toml");
         }
-        #[cfg(debug_assertions)]
+        #[cfg(any(debug_assertions, test, feature = "_enable_seeding"))]
         Command::SeedDatabase(_) => {
             let app = Liwan::try_new(config)?;
-            app.seed_database()?;
+            app.seed_database(100000)?;
             println!("Database seeded with test data");
         }
     }

src/cli.rs

@@ -1,4 +1,3 @@
-#![allow(dead_code)]
 use rand::Rng;
 use time::OffsetDateTime;
 
@@ -44,6 +43,12 @@ pub fn random_events(
         generated += 1;
 
         let created_at = random_date(time_range.0, time_range.1, 0.5);
+
+        // let time_slice = time_range.1 - time_range.0;
+        // let skew_factor = 2.0;
+        // let normalized = 1.0 - (1.0 - (generated as f64 / count as f64)).powf(skew_factor);
+        // let created_at = time_range.0 + time_slice * normalized;
+
         let path = random_el(PATHS, 0.5);
         let referrer = random_el(REFERRERS, 0.5);
         let platform = random_el(PLATFORMS, -0.5);

src/utils/seed.rs

@@ -220,8 +220,10 @@ impl AdminAPI {
             http_bail!(StatusCode::FORBIDDEN, "Forbidden")
         }
 
-        app.users
-            .create(&user.username, &user.password, user.role, &[])
+        let app = app.clone();
+        tokio::task::spawn_blocking(move || app.users.create(&user.username, &user.password, user.role, &[]))
+            .await
+            .http_err("Failed to create user", StatusCode::INTERNAL_SERVER_ERROR)?
             .http_err("Failed to create user", StatusCode::INTERNAL_SERVER_ERROR)?;
 
         EmptyResponse::ok()