ops: add a script to automatically check things before PR creation (#632)

Author: badmonster0

check.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+maturin develop
+mypy
+
+cargo test
+pytest
+
+cargo fmt
+ruff format
+
+echo "All checks passed"

docs/docs/about/contributing.md

@@ -26,36 +26,36 @@ Following the steps below to get cocoindex build on latest codebase locally - if
 -   🦀 [Install Rust](https://rust-lang.org/tools/install)
 
     If you don't have Rust installed, run
-    ```bash
+    ```sh
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     ```
     Already have Rust? Make sure it's up to date 
-    ```bash 
+    ```sh 
     rustup update
     ```
 
--   (Recommended) Setup Python virtual environment:
-    ```bash
+-   Setup Python virtual environment:
+    ```sh
     python3 -m venv .venv
     ```
     Activate the virtual environment, before any installing / building / running:
 
-    ```bash
+    ```sh
     . .venv/bin/activate
     ```
 
--   Install maturin:
-    ```bash
-    pip install maturin
+-   Install required tools:
+    ```sh
+    pip install maturin mypy ruff
     ```
 
 -   Build the library. Run at the root of cocoindex directory:
-    ```bash
+    ```sh
     maturin develop
     ```
 
--   (Optional) Before running a specific example, set extra environment variables, for exposing extra traces, allowing dev UI, etc.
-    ```bash
+-   Before running a specific example, set extra environment variables, for exposing extra traces, allowing dev UI, etc.
+    ```sh
     . ./.env.lib_debug
     ```
 
@@ -67,7 +67,12 @@ To submit your code:
 1. Fork the [CocoIndex repository](https://github.com/cocoindex-io/cocoindex)
 2. [Create a new branch](https://docs.github.com/en/desktop/making-changes-in-a-branch/managing-branches-in-github-desktop) on your fork
 3. Make your changes
-4. [Open a Pull Request (PR)](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) when your work is ready for review
+4. Make sure all tests and linting pass by running
+   ```sh
+   ./check.sh
+   ```
+
+5. [Open a Pull Request (PR)](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) when your work is ready for review
 
 In your PR description, please include:
 - Description of the changes

src/builder/flow_builder.rs

@@ -258,17 +258,14 @@ impl FlowBuilder {
     #[new]
     pub fn new(name: &str) -> PyResult<Self> {
         let lib_context = get_lib_context().into_py_result()?;
-        let existing_flow_ss = lib_context
-            .persistence_ctx
-            .as_ref()
-            .and_then(|ctx| {
-                ctx.all_setup_states
-                    .read()
-                    .unwrap()
-                    .flows
-                    .get(name)
-                    .cloned()
-            });
+        let existing_flow_ss = lib_context.persistence_ctx.as_ref().and_then(|ctx| {
+            ctx.all_setup_states
+                .read()
+                .unwrap()
+                .flows
+                .get(name)
+                .cloned()
+        });
         let root_op_scope = OpScope::new(
             spec::ROOT_SCOPE_NAME.to_string(),
             None,

src/execution/db_tracking_setup.rs

@@ -154,8 +154,6 @@ impl ResourceSetupStatus for TrackingTableSetupStatus {
             (None, None) => SetupChangeType::NoChange,
         }
     }
-
-
 }
 
 impl TrackingTableSetupStatus {

src/execution/dumper.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
-use futures::future::try_join_all;
 use futures::StreamExt;
+use futures::future::try_join_all;
 use indexmap::IndexMap;
 use itertools::Itertools;
 use serde::ser::SerializeSeq;

src/execution/live_updater.rs

@@ -51,11 +51,7 @@ impl SharedAckFn {
         let ack_fn = {
             let mut v = v.lock().unwrap();
             v.count -= 1;
-            if v.count > 0 {
-                None
-            } else {
-                v.ack_fn.take()
-            }
+            if v.count > 0 { None } else { v.ack_fn.take() }
         };
         if let Some(ack_fn) = ack_fn {
             ack_fn().await?;
@@ -108,9 +104,7 @@ async fn update_source(
         } else {
             trace!(
                 "{}.{}: {}",
-                flow_ctx.flow.flow_instance.name,
-                import_op.name,
-                delta
+                flow_ctx.flow.flow_instance.name, import_op.name, delta
             );
         }
     };
@@ -252,10 +246,16 @@ impl FlowLiveUpdater {
         while let Some(result) = self.tasks.join_next().await {
             match result {
                 Err(e) if !e.is_cancelled() => {
-                    error!("A background task in FlowLiveUpdater failed to join: {:?}", e);
+                    error!(
+                        "A background task in FlowLiveUpdater failed to join: {:?}",
+                        e
+                    );
                 }
                 Ok(Err(e)) => {
-                    error!("Error reported by a source update task during live update: {:?}", e);
+                    error!(
+                        "Error reported by a source update task during live update: {:?}",
+                        e
+                    );
                 }
                 _ => {}
             }

src/execution/memoization.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, Result};
+use anyhow::{Result, bail};
 use serde::{Deserialize, Serialize};
 use std::{
     borrow::Cow,

src/execution/source_indexer.rs

@@ -2,7 +2,7 @@ use crate::prelude::*;
 
 use futures::future::Ready;
 use sqlx::PgPool;
-use std::collections::{hash_map, HashMap};
+use std::collections::{HashMap, hash_map};
 use tokio::{sync::Semaphore, task::JoinSet};
 
 use super::{

src/lib_context.rs

@@ -110,7 +110,9 @@ impl LibContext {
             .ok_or_else(|| anyhow!("Database is required for this operation. Please set COCOINDEX_DATABASE_URL environment variable and call cocoindex.init() with database settings."))
     }
 
-    pub fn require_all_setup_states(&self) -> Result<&RwLock<setup::AllSetupState<setup::ExistingMode>>> {
+    pub fn require_all_setup_states(
+        &self,
+    ) -> Result<&RwLock<setup::AllSetupState<setup::ExistingMode>>> {
         self.persistence_ctx
             .as_ref()
             .map(|ctx| &ctx.all_setup_states)
@@ -149,7 +151,7 @@ pub fn create_lib_context(settings: settings::Settings) -> Result<LibContext> {
         // No database configured
         None
     };
-    
+
     Ok(LibContext {
         db_pools,
         persistence_ctx,

src/llm/anthropic.rs

@@ -2,7 +2,7 @@ use crate::llm::{
     LlmGenerateRequest, LlmGenerateResponse, LlmGenerationClient, LlmSpec, OutputFormat,
     ToJsonSchemaOptions,
 };
-use anyhow::{bail, Context, Result};
+use anyhow::{Context, Result, bail};
 use async_trait::async_trait;
 use json5;
 use serde_json::Value;
@@ -115,16 +115,20 @@ impl LlmGenerationClient for Client {
                                 Ok(value) => {
                                     println!("[Anthropic] Used permissive JSON5 parser for output");
                                     serde_json::to_string(&value)?
-                                },
-                                Err(e2) => return Err(anyhow::anyhow!(format!("No structured tool output or text found in response, and permissive JSON5 parsing also failed: {e}; {e2}")))
+                                }
+                                Err(e2) => {
+                                    return Err(anyhow::anyhow!(format!(
+                                        "No structured tool output or text found in response, and permissive JSON5 parsing also failed: {e}; {e2}"
+                                    )));
+                                }
                             }
                         }
                     }
                 }
                 _ => {
                     return Err(anyhow::anyhow!(
                         "No structured tool output or text found in response"
-                    ))
+                    ));
                 }
             }
         };