small tweaks

Author: mProjectsCode

Makefile

@@ -2,12 +2,13 @@ wasm:
 	cd data-wasm && make
 
 data:
-	cd data && make -B
+	cd data && make
 
 format:
 	cd data && make format
 	cd data-lib && make format
 	cd data-wasm && make format
+	cd website && bun run format
 
 lint:
 	cd data && make lint

README.md

@@ -4,6 +4,65 @@ A stats website for the Obsidian Eco-System.
 
 https://www.moritzjung.dev/obsidian-stats/
 
+## Project Structure
+
+The project is split into 4 parts.
+
+1. The website located in `website`
+2. A shared Rust library in `data-lib`
+3. The data collection code in `data`
+4. The data analysis code used for the website in `data-wasm`
+
+All collected data is persisted in the `data/out` folder and tracked in Git, except for cloned plugin repos.
+
+The project uses Makefiles that sometimes think targets are up to date. In that case you can add the `-B` flag to the make command.
+
+### How to Build
+
+Requirements:
+
+1. [Bun](https://bun.com/)
+2. [Latest Rust Compiler](https://rustup.rs/)
+3. [wasm-pack](https://github.com/drager/wasm-pack)
+4. Probably be on Linux, idk
+
+To test and build the website locally you first need to build the `data-wasm` folder. 
+For this you can run `make wasm` in the repo root.
+
+After this you can navigate to the `website` folder and install dependencies with `bun i`.
+Then you can run either the dev server with `bun run dev` or build the website via `bun run build`.
+
+### How to Run Data Collection
+
+Requirements:
+
+1. [Latest Rust Compiler](https://rustup.rs/)
+2. Probably Linux, idk
+3. 10+ minutes of time
+
+Depending on your goal, the first thing you might want to run is `make submodule-update`.
+
+After that you can run the data collection via `make data`.
+
+The data collection consists of multiple phases:
+
+1. Theme Data
+    1. Read theme data from the `obsidian-releases` repo
+    2. Build and save theme data to `data/out/theme-data`
+2. Plugin Data
+    1. Read plugin data from the `obsidian-releases` repo, including download data
+    2. Build plugin, download, and version data
+    3. Save that data to `data/out/plugin-data`
+3. Clone Plugin Repos to `data/out/plugin-repos`
+4. Extract extra data from plugin repos to `data/out/plugin-repo-data`
+5. Process licenses from `choosealicense.com` to `data/out/licenses.json`
+6. Obsidian Releases Data
+    1. Get release data from GitHub releases, process, and save to `data/out/releases-github-raw`. This data is incremental.
+    2. Interpolate download data and save to `data/out/releases-github-interpolated`
+    3. Get the Obsidian changelog, process and save to `data/out/releases-changelog`
+
+Currently you select which steps to run by commenting out the steps that you don't want in `data/src/main.rs`.
+
 ## Credits
 
 I want to thank the following people:

data-lib/src/license.rs

@@ -37,4 +37,4 @@ pub struct Licenses {
     pub conditions: Vec<String>,
     pub limitations: Vec<String>,
     pub descriptions: LicenseDescriptionNested,
-}
+}

data-lib/src/plugin/bundlers.rs

@@ -22,7 +22,7 @@ impl Bundler {
 
     pub fn get_identifier(&self) -> &'static str {
         match self {
-            Bundler::Esbuild => "ESBuild",
+            Bundler::Esbuild => "esbuild",
             Bundler::Rollup => "Rollup",
             Bundler::Webpack => "Webpack",
             Bundler::Vite => "Vite",
@@ -31,12 +31,12 @@ impl Bundler {
     }
 
     pub fn from_identifier(identifier: &str) -> Option<Self> {
-        match identifier {
-            "ESBuild" => Some(Bundler::Esbuild),
-            "Rollup" => Some(Bundler::Rollup),
-            "Webpack" => Some(Bundler::Webpack),
-            "Vite" => Some(Bundler::Vite),
-            "Turbo" => Some(Bundler::Turbo),
+        match identifier.to_lowercase().as_str() {
+            "esbuild" => Some(Bundler::Esbuild),
+            "rollup" => Some(Bundler::Rollup),
+            "webpack" => Some(Bundler::Webpack),
+            "vite" => Some(Bundler::Vite),
+            "turbo" => Some(Bundler::Turbo),
             _ => None,
         }
     }

data-lib/src/plugin/data_array.rs

@@ -4,12 +4,15 @@ use wasm_bindgen::prelude::wasm_bindgen;
 
 use crate::{
     common::{
-        increment_named_data_points, to_percentage, CountMonthlyDataPoint, DownloadDataPoint, HallOfFameDataPoint, InactivityByReleaseDataPoint, IndividualDownloadDataPoint, OverviewDataPoint, RemovedByReleaseDataPoint
+        CountMonthlyDataPoint, DownloadDataPoint, HallOfFameDataPoint,
+        InactivityByReleaseDataPoint, IndividualDownloadDataPoint, OverviewDataPoint,
+        RemovedByReleaseDataPoint, increment_named_data_points, to_percentage,
     },
     date::Date,
     license::Licenses,
     plugin::{
-        full::FullPluginData, LicenseInfo, NamedDataPoint, PluginData, PluginExtraData, PluginLicenseDataPoints, PluginRepoDataPoints
+        LicenseInfo, NamedDataPoint, PluginData, PluginExtraData, PluginLicenseDataPoints,
+        PluginRepoDataPoints, full::FullPluginData,
     },
 };
 
@@ -512,13 +515,14 @@ impl PluginDataArrayView {
 
             match &repo_data.file_license {
                 LicenseInfo::Known(name) => {
-                    let license_data = licenses
-                        .licenses
-                        .iter()
-                        .find(|l| *name == l.spdx_id);
+                    let license_data = licenses.licenses.iter().find(|l| *name == l.spdx_id);
 
                     if let Some(license_data) = license_data {
-                        increment_named_data_points(&mut points.licenses, &license_data.spdx_id, 1.0);
+                        increment_named_data_points(
+                            &mut points.licenses,
+                            &license_data.spdx_id,
+                            1.0,
+                        );
 
                         for permission in &license_data.permissions {
                             increment_named_data_points(&mut points.permissions, permission, 1.0);
@@ -531,11 +535,19 @@ impl PluginDataArrayView {
                         }
                     } else {
                         // I think we should never hit this path, as we make sure that the license is known during data extraction.
-                        increment_named_data_points(&mut points.licenses, &LicenseInfo::Unrecognized.to_fancy_string(), 1.0);
+                        increment_named_data_points(
+                            &mut points.licenses,
+                            &LicenseInfo::Unrecognized.to_fancy_string(),
+                            1.0,
+                        );
                     }
                 }
                 other => {
-                    increment_named_data_points(&mut points.licenses, &other.to_fancy_string(), 1.0);
+                    increment_named_data_points(
+                        &mut points.licenses,
+                        &other.to_fancy_string(),
+                        1.0,
+                    );
                 }
             }
         });

data-lib/src/plugin/full.rs

@@ -5,7 +5,8 @@ use crate::{
     common::{DownloadDataPoint, EntryChangeDataPoint, VersionDataPoint},
     date::Date,
     plugin::{
-        warnings::{get_plugin_warnings, PluginWarning}, FundingUrl, LicenseInfo, PluginData, PluginExtraData, PluginRepoData
+        FundingUrl, LicenseInfo, PluginData, PluginExtraData, PluginRepoData,
+        warnings::{PluginWarning, get_plugin_warnings},
     },
 };
 

data-lib/src/plugin/mod.rs

@@ -151,9 +151,8 @@ impl From<Option<LicenseInfo>> for LicenseInfo {
     }
 }
 
-
 impl From<Option<&LicenseInfo>> for LicenseInfo {
     fn from(value: Option<&LicenseInfo>) -> Self {
         value.cloned().unwrap_or(LicenseInfo::NotFound)
     }
-}
+}

data-lib/src/plugin/warnings.rs

@@ -1,7 +1,11 @@
 use serde::Serialize;
 use tsify::Tsify;
 
-use crate::{commit::StringCommit, date::Date, plugin::{full::FullPluginData, LicenseInfo}};
+use crate::{
+    commit::StringCommit,
+    date::Date,
+    plugin::{LicenseInfo, full::FullPluginData},
+};
 
 #[derive(Tsify, Debug, Clone, Serialize)]
 #[tsify(into_wasm_abi)]

data-lib/src/release/mod.rs

@@ -134,7 +134,7 @@ pub struct ChangelogDataPoint {
     pub number_of_patches: usize,
 }
 
-#[derive(Tsify, Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)]
+#[derive(Tsify, Debug, Clone, Hash, PartialEq, Eq)]
 #[tsify(into_wasm_abi)]
 pub enum ChangeLogChangeCategory {
     BreakingChange,
@@ -171,6 +171,15 @@ impl ChangeLogChangeCategory {
     }
 }
 
+impl Serialize for ChangeLogChangeCategory {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.to_fancy_string())
+    }
+}
+
 impl From<Option<ChangeLogChangeCategory>> for ChangeLogChangeCategory {
     fn from(value: Option<ChangeLogChangeCategory>) -> Self {
         value.unwrap_or(ChangeLogChangeCategory::Uncategorized)
@@ -209,7 +218,7 @@ impl From<&str> for ChangeLogChangeCategory {
     }
 }
 
-#[derive(Tsify, Debug, Clone, Serialize, Deserialize)]
+#[derive(Tsify, Debug, Clone, Serialize)]
 #[tsify(into_wasm_abi)]
 pub struct ChangelogChanges {
     pub version: Version,

data/src/plugins/extra.rs

@@ -3,7 +3,8 @@ use std::{fs, path::Path};
 use data_lib::{
     input_data::{ObsCommunityPluginDeprecations, ObsCommunityPluginRemoved},
     plugin::{
-        bundlers::Bundler, packages::PackageManager, testing::TestingFramework, LicenseInfo, PluginData, PluginExtraData, PluginRepoData
+        LicenseInfo, PluginData, PluginExtraData, PluginRepoData, bundlers::Bundler,
+        packages::PackageManager, testing::TestingFramework,
     },
 };
 use hashbrown::HashMap;
@@ -156,11 +157,13 @@ pub fn extract_data_from_repo(
             || lower_case_file == "license.md"
     });
 
-    let file_license = license_file.and_then(|file| {
-        fs::read_to_string(format!("{repo_path}/{file}")).map(|license_text| {
-            license_comparer.compare(&plugin.id, &license_text)
-        }).ok()
-    }).into();
+    let file_license = license_file
+        .and_then(|file| {
+            fs::read_to_string(format!("{repo_path}/{file}"))
+                .map(|license_text| license_comparer.compare(&plugin.id, &license_text))
+                .ok()
+        })
+        .into();
 
     Ok(PluginRepoData {
         uses_typescript,