refactor: clean up code, prevent freezing in the sim

probably some other stuff in here too... lesson to self: make smaller commits

Author: pepperoni505

PackageSources/SimObjects/Airplanes/Navigraph_Navdata_Updater_Aircraft/panel/navdata_updater.wasm

@@ -29,6 +29,13 @@ export class NavigraphLogin extends DisplayComponent<NavigraphLoginProps> {
       console.info("WASM downloaded navdata")
       this.navdataTextRef.instance.textContent = "Navdata downloaded!"
     })
+
+    this.commBusListener.on("UnzippedFilesRemaining", (jsonArgs: string) => {
+      const args = JSON.parse(jsonArgs)
+      console.info("WASM unzipping files", args)
+      const percent = Math.round((args.unzipped / args.total) * 100)
+      this.navdataTextRef.instance.textContent = `Unzipping files... ${percent}% complete`
+    })
   }
 
   public render(): VNode {

Sources/Code/TypeScript/Components/NavigraphLogin.tsx

@@ -1,9 +1,2 @@
-[target.wasm32-wasi]
-rustflags = [
-  "-Clink-arg=--export-table",
-  "-Clink-arg=--export=malloc",
-  "-Clink-arg=--export=free",
-]
-
 [build]
 target = "wasm32-wasi"

Sources/Code/WASM/navdata_updater/.cargo/config.toml

@@ -17,8 +17,5 @@ strip = true
 msfs = { git = "https://github.com/pepperoni505/msfs-rs.git", branch = "main" }
 serde = "1.0.190"
 serde_json = "1.0.108"
-rusqlite = { git = "https://github.com/rusqlite/rusqlite", rev = "d7ac6a1", features = ["bundled", "wasm32-wasi-vfs"] }
-zip = { version = "0.6.4", default-features = false, features = ["deflate"] }
-
-[patch.crates-io]
-libsqlite3-sys = { git = "https://github.com/navigraph/rusqlite", branch = "master" }
+rusqlite = { git = "https://github.com/navigraph/rusqlite", rev = "366c520", features = ["bundled", "wasm32-wasi-vfs"] }
+zip = { version = "0.6.4", default-features = false, features = ["deflate"] }

Sources/Code/WASM/navdata_updater/Cargo.toml

@@ -1,13 +1,2 @@
-@echo off
-
-set "CLANG_LIB_WASI=%WASI_SDK%\\lib\\clang\\15.0.7\\lib\\wasi"
-set "WASI_SYSROOT=%WASI_SDK%\\share\\wasi-sysroot"
-set "CC=%WASI_SDK%\\bin\\clang --sysroot=%WASI_SYSROOT%"
-set "AR=%WASI_SDK%\\bin\\llvm-ar"
-set "CC_wasm32_wasi=%WASI_SDK%\\bin\\clang"
-set "LIBSQLITE3_FLAGS=-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_SHARED_CACHE -D_LARGEFILE64_SOURCE"
-set "RUSTFLAGS=-Clink-arg=--export-table -Clink-arg=--export=malloc -Clink-arg=--export=free -Clink-arg=-L%CLANG_LIB_WASI% -Clink-arg=-lclang_rt.builtins-wasm32"
-
-
-cargo build --release
+call run_cargo_cmd.bat build --release
 copy "target\wasm32-wasi\release\navdata_updater.wasm" "..\..\..\..\PackageSources\SimObjects\Airplanes\Navigraph_Navdata_Updater_Aircraft\panel\navdata_updater.wasm"

Sources/Code/WASM/navdata_updater/build.bat

@@ -0,0 +1,17 @@
+@echo off
+
+set "CLANG_LIB_WASI=%WASI_SDK%\\lib\\clang\\15.0.7\\lib\\wasi"
+set "WASI_SYSROOT=%WASI_SDK%\\share\\wasi-sysroot"
+set "CC=%WASI_SDK%\\bin\\clang --sysroot=%WASI_SYSROOT%"
+set "AR=%WASI_SDK%\\bin\\llvm-ar"
+set "CC_wasm32_wasi=%WASI_SDK%\\bin\\clang"
+set "LIBSQLITE3_FLAGS=-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_SHARED_CACHE -D_LARGEFILE64_SOURCE"
+set "RUSTFLAGS=-Clink-arg=-L%CLANG_LIB_WASI% -Clink-arg=-lclang_rt.builtins-wasm32 -Clink-arg=--export-table -Clink-arg=--export=malloc -Clink-arg=--export=free"
+
+
+:: Run the Cargo command passed as an argument
+if "%1"=="" (
+    echo No Cargo command specified.
+) else (
+    cargo %*
+)

Sources/Code/WASM/navdata_updater/run_cargo_cmd.bat

@@ -0,0 +1,77 @@
+use std::collections::HashMap;
+use std::mem;
+use std::rc::Rc;
+
+use crate::download::downloader::NavdataDownloader;
+use msfs::{commbus::*, MSFSEvent};
+
+pub struct Dispatcher<'a> {
+    commbus: Option<CommBus<'a>>,
+    downloader: Rc<NavdataDownloader>,
+}
+
+impl<'a> Dispatcher<'a> {
+    pub fn new() -> Self {
+        Dispatcher {
+            commbus: None,
+            downloader: Rc::new(NavdataDownloader::new()),
+        }
+    }
+
+    pub fn on_msfs_event(&mut self, event: MSFSEvent) {
+        match event {
+            MSFSEvent::PostInitialize => {
+                self.handle_initialized();
+            }
+            MSFSEvent::PostUpdate => {
+                self.handle_update();
+            }
+            MSFSEvent::PreKill => {
+                // handle pre kill TODO wait for the unregister functions to be ported
+            }
+
+            _ => {}
+        }
+    }
+
+    fn handle_initialized(&mut self) {
+        println!("[WASM] Initialized");
+        let captured_downloader = self.downloader.clone();
+        self.commbus = CommBus::register("DownloadNavdata", move |args| {
+            captured_downloader.download(args)
+        });
+    }
+
+    fn handle_update(&mut self) {
+        // update unzip
+        // todo: maybe another way to check instead of cloning? i mean we drop the value anyway but not sure on performance
+        let captured_downloader = self.downloader.clone();
+        if captured_downloader.get_files_to_unzip() > 0 {
+            let total_files = captured_downloader.get_total_files();
+            let files_unzipped = captured_downloader.get_files_unzipped();
+            let mut map = HashMap::new();
+            map.insert("total", total_files);
+            map.insert("unzipped", files_unzipped);
+            let data = serde_json::to_string(&map).unwrap();
+            // this is temporary until msfs-rs handles this unsafe stuff (soon TM)
+            let i8_slice: &[i8] = unsafe { mem::transmute(data.as_bytes()) };
+            println!(
+                "[WASM] total: {}, unzipped: {}",
+                total_files, files_unzipped
+            );
+            // only send the call if unzipped is divisible by 100 (kinda hacky but otherwise we flood the commbus (not good!))
+            if files_unzipped % 100 == 0 {
+                CommBus::call(
+                    "UnzippedFilesRemaining",
+                    i8_slice,
+                    CommBusBroadcastFlags::JS,
+                );
+            }
+            let has_more_files = captured_downloader.unzip_batch(10);
+            if !has_more_files {
+                println!("[WASM] finished unzip");
+                CommBus::call("NavdataDownloaded", &[], CommBusBroadcastFlags::JS);
+            }
+        }
+    }
+}

Sources/Code/WASM/navdata_updater/src/dispatcher.rs

@@ -0,0 +1,100 @@
+use std::cell::RefCell;
+use std::fs;
+use std::io::Cursor;
+use std::path::PathBuf;
+use std::rc::Rc;
+
+use msfs::network::*;
+
+use crate::download::zip_handler::ZipFileHandler;
+use crate::util::JsonParser;
+
+pub struct NavdataDownloader {
+    zip_handler: RefCell<Option<ZipFileHandler<Cursor<Vec<u8>>>>>,
+}
+
+impl NavdataDownloader {
+    pub fn new() -> Self {
+        NavdataDownloader {
+            zip_handler: RefCell::new(None),
+        }
+    }
+
+    pub fn download(self: &Rc<Self>, args: &[u8]) {
+        println!("[WASM] call received");
+        let json_result = JsonParser::parse(args);
+        if json_result.is_err() {
+            println!("[WASM] json error: {}", json_result.err().unwrap());
+            return;
+        }
+        let json = json_result.unwrap();
+        let url = json["url"].as_str().unwrap_or_default();
+
+        let captured_self = self.clone();
+        NetworkRequestBuilder::new(url)
+            .unwrap()
+            .with_callback(move |request, status_code| {
+                captured_self.request_finished_callback(request, status_code)
+            })
+            .get()
+            .unwrap();
+    }
+
+    fn request_finished_callback(&self, request: NetworkRequest, status_code: i32) {
+        if status_code != 200 {
+            println!("[WASM] request failed");
+            return;
+        }
+        let path = PathBuf::from("\\work/navdata");
+        if let Err(e) = fs::create_dir_all(&path) {
+            println!("[WASM] directory error: {}", e);
+            return;
+        }
+
+        let data = request.data().unwrap();
+        let cursor = Cursor::new(data);
+        let zip = zip::ZipArchive::new(cursor).unwrap();
+
+        let handler = ZipFileHandler::new(zip, path);
+        let mut zip_handler = self.zip_handler.borrow_mut();
+        *zip_handler = Some(handler);
+    }
+
+    /// Returns the number of files left to unzip
+    pub fn get_files_to_unzip(&self) -> usize {
+        let zip_handler = self.zip_handler.borrow();
+        match zip_handler.as_ref() {
+            Some(handler) => handler.zip_file_count - handler.current_file_index,
+            None => 0,
+        }
+    }
+
+    /// Returns the total number of files in the zip
+    pub fn get_total_files(&self) -> usize {
+        let zip_handler = self.zip_handler.borrow();
+        match zip_handler.as_ref() {
+            Some(handler) => handler.zip_file_count,
+            None => 0,
+        }
+    }
+
+    /// Returns the number of files that have been unzipped
+    pub fn get_files_unzipped(&self) -> usize {
+        let zip_handler = self.zip_handler.borrow();
+        match zip_handler.as_ref() {
+            Some(handler) => handler.current_file_index,
+            None => 0,
+        }
+    }
+
+    /// Unzips a batch of files
+    ///
+    /// Returns true if there are more files to unzip (false if we are done)
+    pub fn unzip_batch(&self, batch_size: usize) -> bool {
+        let mut zip_handler = self.zip_handler.borrow_mut();
+        match zip_handler.as_mut() {
+            Some(handler) => handler.unzip_batch(batch_size),
+            None => false,
+        }
+    }
+}

Sources/Code/WASM/navdata_updater/src/download/downloader.rs

@@ -0,0 +1,2 @@
+pub mod downloader;
+pub mod zip_handler;

Sources/Code/WASM/navdata_updater/src/download/mod.rs

@@ -0,0 +1,61 @@
+use std::fs;
+use std::io;
+use std::path::PathBuf;
+
+pub struct ZipFileHandler<R: io::Read + io::Seek> {
+    pub zip_archive: Option<zip::ZipArchive<R>>,
+    path_buf: PathBuf,
+    pub current_file_index: usize,
+    pub zip_file_count: usize,
+}
+
+impl<R: io::Read + io::Seek> ZipFileHandler<R> {
+    pub fn new(zip_archive: zip::ZipArchive<R>, path_buf: PathBuf) -> Self {
+        // To make accessing zip archive size easier, we just store it to the struct instead of calling it every time (avoids ownership issues)
+        let zip_file_count = zip_archive.len();
+        Self {
+            zip_archive: Some(zip_archive),
+            path_buf,
+            current_file_index: 0,
+            zip_file_count,
+        }
+    }
+
+    pub fn unzip_batch(&mut self, batch_size: usize) -> bool {
+        if self.zip_archive.is_none() {
+            return false;
+        }
+        let unwrapped_zip_archive = self.zip_archive.as_mut().unwrap();
+        let total_files = unwrapped_zip_archive.len();
+        for _ in 0..batch_size {
+            if self.current_file_index >= total_files {
+                // lets clear our zip
+                self.zip_archive = None;
+                return false; // no more files to unzip
+            }
+
+            let mut file = match unwrapped_zip_archive.by_index(self.current_file_index) {
+                Ok(file) => file,
+                Err(_) => continue,
+            };
+            let outpath = match file.enclosed_name() {
+                Some(path) => self.path_buf.join(path),
+                None => continue,
+            };
+
+            if (*file.name()).ends_with('/') {
+                fs::create_dir_all(outpath).unwrap();
+            } else {
+                if let Some(p) = outpath.parent() {
+                    if !p.exists() {
+                        fs::create_dir_all(p).unwrap();
+                    }
+                }
+                let mut outfile = fs::File::create(outpath).unwrap();
+                io::copy(&mut file, &mut outfile).unwrap();
+            }
+            self.current_file_index += 1;
+        }
+        true
+    }
+}