feat: Revamp Makefile for streamlined development workflow and enhance test utilities with new macros and schema handling

Author: longshan.lu

Makefile

@@ -1,6 +1,174 @@
-check:
-	cargo check --all-features
+# Qurious Makefile
+# Simplified development workflow
+CARGO = cargo
+RUSTFMT = rustfmt
+DOCKER = docker
+PROJECT_NAME = qurious
+TPCH_DATA_DIR = qurious/tests/tpch/data
+TPCH_DOCKER_IMAGE = ghcr.io/scalytics/tpch-docker:main
 
-make tpch:
-	mkdir -p qurious/tests/tpch/data
-	docker run -it -v "$(realpath qurious/tests/tpch/data)":/data ghcr.io/scalytics/tpch-docker:main -vf -s 0.01
+# Default target
+.PHONY: help
+help:
+	@echo "Qurious Development Tools"
+	@echo "========================"
+	@echo ""
+	@echo "Available commands:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+# Code checking
+.PHONY: check
+check: ## Check code syntax and dependencies
+	$(CARGO) check --all-features
+
+.PHONY: check-all
+check-all: ## Check all workspace members
+	$(CARGO) check --workspace --all-features
+
+# Build
+.PHONY: build
+build: ## Build project (debug mode)
+	$(CARGO) build
+
+.PHONY: build-release
+build-release: ## Build project (release mode)
+	$(CARGO) build --release
+
+.PHONY: build-all
+build-all: ## Build all workspace members
+	$(CARGO) build --workspace
+
+.PHONY: test
+test: ## Run unit tests
+	INCLUDE_TPCH=true $(CARGO) test
+
+# Code formatting
+.PHONY: fmt
+fmt: ## Check code formatting
+	$(CARGO) fmt -- --check
+
+.PHONY: fmt-fix
+fmt-fix: ## Format code and auto-fix
+	$(CARGO) fmt
+
+# Code quality
+.PHONY: clippy
+clippy: ## Run clippy code checks
+	$(CARGO) clippy --all-features -- -D warnings
+
+.PHONY: clippy-fix
+clippy-fix: ## Run clippy and auto-fix
+	$(CARGO) clippy --fix --all-features
+
+# Clean
+.PHONY: clean
+clean: ## Clean build artifacts
+	$(CARGO) clean
+
+.PHONY: clean-all
+clean-all: clean ## Clean all build artifacts and temp files
+	rm -rf target/
+	find . -name "*.orig" -delete
+	find . -name "*.rej" -delete
+
+# TPC-H data generation
+.PHONY: tpch-data
+tpch-data: ## Generate TPC-H test data
+	mkdir -p $(TPCH_DATA_DIR)
+	$(DOCKER) run -it -v "$(realpath $(TPCH_DATA_DIR))":/data $(TPCH_DOCKER_IMAGE) -vf -s 0.01
+
+.PHONY: tpch-data-small
+tpch-data-small: ## Generate small TPC-H test data
+	mkdir -p $(TPCH_DATA_DIR)
+	$(DOCKER) run -it -v "$(realpath $(TPCH_DATA_DIR))":/data $(TPCH_DOCKER_IMAGE) -vf -s 0.001
+
+.PHONY: tpch-data-large
+tpch-data-large: ## Generate large TPC-H test data
+	mkdir -p $(TPCH_DATA_DIR)
+	$(DOCKER) run -it -v "$(realpath $(TPCH_DATA_DIR))":/data $(TPCH_DOCKER_IMAGE) -vf -s 0.1
+
+# Development workflow
+.PHONY: dev-setup
+dev-setup: ## Setup development environment
+	rustup component add rustfmt
+	rustup component add clippy
+	$(CARGO) install cargo-watch
+
+.PHONY: dev
+dev: ## Development mode: watch files and run tests
+	cargo watch -x check -x test
+
+.PHONY: dev-test
+dev-test: ## Development mode: watch files and run tests
+	cargo watch -x test
+
+# Documentation
+.PHONY: doc
+doc: ## Generate documentation
+	$(CARGO) doc --no-deps
+
+.PHONY: doc-open
+doc-open: ## Generate and open documentation
+	$(CARGO) doc --no-deps --open
+
+# Benchmark
+.PHONY: bench
+bench: ## Run benchmarks
+	$(CARGO) bench
+
+# Dependency management
+.PHONY: update
+update: ## Update dependencies
+	$(CARGO) update
+
+.PHONY: audit
+audit: ## Check dependency security vulnerabilities
+	$(CARGO) audit
+
+# Release preparation
+.PHONY: release-check
+release-check: check-all clippy test-all ## Pre-release checks
+	@echo "All checks passed, ready to release!"
+
+.PHONY: release-build
+release-build: ## Build release version
+	$(CARGO) build --release
+	@echo "Release build completed: target/release/$(PROJECT_NAME)"
+
+# Database
+.PHONY: db-start
+db-start: ## Start database services
+	$(DOCKER) compose up -d
+
+.PHONY: db-stop
+db-stop: ## Stop database services
+	$(DOCKER) compose down
+
+.PHONY: db-reset
+db-reset: ## Reset database
+	$(DOCKER) compose down -v
+	$(DOCKER) compose up -d
+
+# Utilities
+.PHONY: size
+size: build-release ## Show binary file size
+	@echo "Binary file size:"
+	@ls -lh target/release/$(PROJECT_NAME)
+
+.PHONY: deps-tree
+deps-tree: ## Show dependency tree
+	$(CARGO) tree
+
+.PHONY: outdated
+outdated: ## Check outdated dependencies
+	$(CARGO) install-update -a
+
+# Quick development command combinations
+.PHONY: quick-check
+quick-check: fmt clippy test ## Quick check: format, clippy, test
+
+.PHONY: full-check
+full-check: fmt clippy test-all audit ## Full check: format, clippy, all tests, security audit
+
+# Default goal
+.DEFAULT_GOAL := help

qurious/src/common/table_relation.rs

@@ -76,7 +76,7 @@ impl TableRelation {
     }
 
     /// Return the fully qualified name of the table
-    pub fn to_quanlify_name(&self) -> String {
+    pub fn to_qualified_name(&self) -> String {
         if self.is_file_source {
             let identify = self.identify.as_ref().expect("should have identify");
             return format!("tmp_table({})", &identify[..7]);
@@ -96,7 +96,7 @@ impl TableRelation {
 
 impl Display for TableRelation {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.to_quanlify_name().fmt(f)
+        self.to_qualified_name().fmt(f)
     }
 }
 
@@ -144,18 +144,18 @@ mod tests {
     #[test]
     fn test_table_relation() {
         let table = TableRelation::from("table".to_string());
-        assert_eq!(table.to_quanlify_name(), "table");
+        assert_eq!(table.to_qualified_name(), "table");
 
         let table = TableRelation::from("schema.table".to_string());
-        assert_eq!(table.to_quanlify_name(), "schema.table");
+        assert_eq!(table.to_qualified_name(), "schema.table");
 
         let table = TableRelation::from("catalog.schema.table".to_string());
-        assert_eq!(table.to_quanlify_name(), "catalog.schema.table");
+        assert_eq!(table.to_qualified_name(), "catalog.schema.table");
     }
 
     #[test]
     fn test_parse_file_path() {
         let table = TableRelation::parse_file_path("./tests/testdata/file/case1.csv");
-        assert_eq!(table.to_quanlify_name(), "tmp_table(b563e59)");
+        assert_eq!(table.to_qualified_name(), "tmp_table(b563e59)");
     }
 }

qurious/src/common/table_schema.rs

@@ -6,7 +6,7 @@ use crate::{
     internal_err,
     logical::expr::Column,
 };
-use arrow::datatypes::{Field, FieldRef, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};
 
 pub type TableSchemaRef = Arc<TableSchema>;
 
@@ -73,6 +73,21 @@ impl TableSchema {
             .zip(self.schema.fields().iter())
             .map(|(q, f)| (q.as_ref(), f))
     }
+
+    pub fn data_type_and_nullable(&self, relation: Option<&TableRelation>, name: &str) -> Result<(DataType, bool)> {
+        let index = self
+            .schema
+            .index_of(name)
+            .map_err(|e| Error::InternalError(format!("Field [{}] not found in schema, error: {}", name, e)))?;
+
+        if self.field_qualifiers[index].as_ref() != relation {
+            return internal_err!("Field [{}] is not qualified with [{}]", name, relation.unwrap());
+        }
+
+        let field = self.schema.field(index);
+
+        Ok((field.data_type().clone(), field.is_nullable()))
+    }
 }
 
 impl TableSchema {

qurious/src/datatypes/operator.rs

@@ -8,7 +8,7 @@ pub enum Operator {
     GtEq,
     Lt,
     LtEq,
-    
+
     And,
     Or,
 

qurious/src/datatypes/scalar.rs

@@ -1,7 +1,9 @@
 use crate::error::{Error, Result};
 use arrow::{
     array::{
-        new_null_array, Array, ArrayRef, ArrowPrimitiveType, BooleanArray, Decimal128Array, Decimal256Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, PrimitiveArray, StringArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array
+        new_null_array, Array, ArrayRef, ArrowPrimitiveType, BooleanArray, Decimal128Array, Decimal256Array,
+        Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, PrimitiveArray,
+        StringArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
     },
     datatypes::{i256, DataType, Field},
 };

qurious/src/execution/mod.rs

@@ -1,5 +1,5 @@
 pub mod session;
 
 mod config;
+mod information_schema;
 mod providers;
-mod information_schema;

qurious/src/execution/session.rs

@@ -195,7 +195,7 @@ impl ExecuteSession {
             .table(table.table())
             .ok_or(Error::InternalError(format!(
                 "failed to resolve table: {}",
-                table.to_quanlify_name()
+                table.to_qualified_name()
             )))
     }
 
@@ -204,12 +204,12 @@ impl ExecuteSession {
             .catalog(table.catalog().unwrap_or(&self.config.default_catalog))
             .ok_or(Error::InternalError(format!(
                 "failed to resolve catalog: {}",
-                table.to_quanlify_name()
+                table.to_qualified_name()
             )))?
             .schema(table.schema().unwrap_or(&self.config.default_schema))
             .ok_or(Error::PlanError(format!(
                 "failed to resolve schema: {}",
-                table.to_quanlify_name()
+                table.to_qualified_name()
             )))
     }
 
@@ -237,7 +237,7 @@ impl ExecuteSession {
                 } else {
                     Err(Error::PlanError(format!(
                         "Drop table failed, table not found: {}",
-                        table.to_quanlify_name()
+                        table.to_qualified_name()
                     )))
                 }
             }
@@ -380,31 +380,54 @@ mod tests {
         // session.sql("INSERT INTO test VALUES (1, 1), (2, 2), (3, 3), (3, 5), (NULL, NULL);")?;
         // session.sql("select a, b, c, d from x join y on a = c")?;
         println!("++++++++++++++");
-        let batch = session.sql(
-            "select
-    l_orderkey,
-    sum(l_extendedprice * (1 - l_discount)) as revenue,
-    o_orderdate,
-    o_shippriority
+        let batch = session
+            .sql(
+                "select
+    s_acctbal,
+    s_name,
+    n_name,
+    p_partkey,
+    p_mfgr,
+    s_address,
+    s_phone,
+    s_comment
 from
-    customer,
-    orders,
-    lineitem
+    part,
+    supplier,
+    partsupp,
+    nation,
+    region
 where
-        c_mktsegment = 'BUILDING'
-  and c_custkey = o_custkey
-  and l_orderkey = o_orderkey
-  and o_orderdate < date '1995-03-15'
-  and l_shipdate > date '1995-03-15'
-group by
-    l_orderkey,
-    o_orderdate,
-    o_shippriority
+        p_partkey = ps_partkey
+  and s_suppkey = ps_suppkey
+  and p_size = 15
+  and p_type like '%BRASS'
+  and s_nationkey = n_nationkey
+  and n_regionkey = r_regionkey
+  and r_name = 'EUROPE'
+  and ps_supplycost = (
+    select
+        min(ps_supplycost)
+    from
+        partsupp,
+        supplier,
+        nation,
+        region
+    where
+            p_partkey = ps_partkey
+      and s_suppkey = ps_suppkey
+      and s_nationkey = n_nationkey
+      and n_regionkey = r_regionkey
+      and r_name = 'EUROPE'
+)
 order by
-    revenue desc,
-    o_orderdate
- limit 10;",
-        )?;
+    s_acctbal desc,
+    n_name,
+    s_name,
+    p_partkey
+limit 10;",
+            )
+            .unwrap();
 
         // let batch = session.sql("select avg(l_quantity) as count_order from lineitem")?;