From 4f9e3b5a6fda94f98a3689b5433910b2f9342050 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Tue, 29 Jul 2025 15:49:50 +0100 Subject: [PATCH 1/4] Speedup new backend Speedup new backend by using only one thread and a custom memory allocator Copied from https://github.com/egraphs-good/egglog/pull/637 --- Cargo.lock | 52 +++++++++++++++++++++++++++++++++++++++------- Cargo.toml | 13 ++++++------ src/conversions.rs | 7 ++++--- src/egraph.rs | 6 +++--- src/lib.rs | 8 +++++++ 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d799fcc3..25d2a401 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,7 +5,7 @@ version = 4 [[package]] name = "add_primitive" version = "0.1.0" -source = "git+https://github.com/saulshanabrook/egg-smol.git?rev=1e638e6#1e638e63cd05f16ed9b7d40bd7b8e83fb81d580b" +source = "git+https://github.com/egraphs-good/egglog.git?rev=5542549#55425498b92bab18fcf3ea35224e42e2ad0afff6" dependencies = [ "quote", "syn 2.0.100", @@ -124,6 +124,15 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "cc" +version = "1.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -188,7 +197,7 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "concurrency" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=1a1f913#1a1f913e88101dbd6f98cc2468be0f0d1b87960b" +source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=cd51d04#cd51d048f9ef8e0bd9eac2966c35e054ac5c8368" dependencies = [ "arc-swap", "rayon", @@ -197,7 +206,7 @@ dependencies = [ [[package]] name = "core-relations" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=1a1f913#1a1f913e88101dbd6f98cc2468be0f0d1b87960b" +source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=cd51d04#cd51d048f9ef8e0bd9eac2966c35e054ac5c8368" dependencies = [ "anyhow", "bumpalo", @@ -346,7 +355,7 @@ checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" [[package]] name = "egglog" version = "0.5.0" -source = "git+https://github.com/saulshanabrook/egg-smol.git?rev=1e638e6#1e638e63cd05f16ed9b7d40bd7b8e83fb81d580b" +source = "git+https://github.com/egraphs-good/egglog.git?rev=5542549#55425498b92bab18fcf3ea35224e42e2ad0afff6" dependencies = [ "add_primitive", "chrono", @@ -372,7 +381,7 @@ dependencies = [ [[package]] name = "egglog-bridge" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=1a1f913#1a1f913e88101dbd6f98cc2468be0f0d1b87960b" +source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=cd51d04#cd51d048f9ef8e0bd9eac2966c35e054ac5c8368" dependencies = [ "anyhow", "core-relations", @@ -394,7 +403,7 @@ dependencies = [ [[package]] name = "egglog-experimental" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-experimental?rev=202078f#202078fee12499f0160293f172546548f909b3e3" +source = "git+https://github.com/egraphs-good/egglog-experimental?rev=255b67a#255b67afaaac6436fee57cc2b9f4650e00c3408c" dependencies = [ "egglog", "lazy_static", @@ -413,9 +422,11 @@ dependencies = [ "egraph-serialize", "lalrpop-util", "log", + "mimalloc", "ordered-float", "pyo3", "pyo3-log", + "rayon", "serde_json", "uuid", ] @@ -696,6 +707,16 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libmimalloc-sys" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -733,6 +754,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mimalloc" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "num" version = "0.4.3" @@ -809,7 +839,7 @@ dependencies = [ [[package]] name = "numeric-id" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=1a1f913#1a1f913e88101dbd6f98cc2468be0f0d1b87960b" +source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=cd51d04#cd51d048f9ef8e0bd9eac2966c35e054ac5c8368" dependencies = [ "lazy_static", "rayon", @@ -1202,6 +1232,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "sized-chunks" version = "0.6.5" @@ -1332,7 +1368,7 @@ checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "union-find" version = "0.1.0" -source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=1a1f913#1a1f913e88101dbd6f98cc2468be0f0d1b87960b" +source = "git+https://github.com/egraphs-good/egglog-backend.git?rev=cd51d04#cd51d048f9ef8e0bd9eac2966c35e054ac5c8368" dependencies = [ "concurrency", "crossbeam", diff --git a/Cargo.toml b/Cargo.toml index 9d38bb8d..9ec631e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,12 +12,11 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.24.2", features = ["extension-module"] } -# include this bugfix https://github.com/egraphs-good/egglog/pull/629 -egglog = { git = "https://github.com/saulshanabrook/egg-smol.git", rev = "1e638e6" } +egglog = { git = "https://github.com/egraphs-good/egglog.git", rev = "5542549" } # egglog = { path = "../egg-smol" } -egglog-bridge = { git = "https://github.com/egraphs-good/egglog-backend.git", rev = "1a1f913" } -core-relations = { git = "https://github.com/egraphs-good/egglog-backend.git", rev = "1a1f913" } -egglog-experimental = { git = "https://github.com/egraphs-good/egglog-experimental", rev = "202078f" } +egglog-bridge = { git = "https://github.com/egraphs-good/egglog-backend.git", rev = "cd51d04" } +core-relations = { git = "https://github.com/egraphs-good/egglog-backend.git", rev = "cd51d04" } +egglog-experimental = { git = "https://github.com/egraphs-good/egglog-experimental", rev = "255b67a" } egraph-serialize = { version = "0.2.0", features = ["serde", "graphviz"] } serde_json = "1.0.140" pyo3-log = "0.12.4" @@ -25,12 +24,14 @@ log = "0.4.27" lalrpop-util = { version = "0.22", features = ["lexer"] } ordered-float = "3.7.0" uuid = { version = "1.17.0", features = ["v4"] } +rayon = "1.10.0" +mimalloc = "0.1.46" # Use patched version of egglog in experimental [patch.'https://github.com/egraphs-good/egglog'] # egglog = { git = "https://github.com/egraphs-good//egglog.git", rev = "d2fa5b733de0796fb187dc5a27e570d5644aa75a" } # egglog = { path = "../egg-smol" } -egglog = { git = "https://github.com/saulshanabrook/egg-smol.git", rev = "1e638e6" } +# egglog = { git = "https://github.com/egraphs-good//egglog.git", rev = "5542549" } # enable debug symbols for easier profiling [profile.release] diff --git a/src/conversions.rs b/src/conversions.rs index 443269b2..4a62acf0 100644 --- a/src/conversions.rs +++ b/src/conversions.rs @@ -1,6 +1,7 @@ // Create wrappers around input types so that convert from pyobjects to them // and then from them to the egg_smol types use crate::utils::*; +use egglog::extract::DefaultCost; use ordered_float::OrderedFloat; use pyo3::prelude::*; use pyo3::types::{PyDelta, PyDeltaAccess}; @@ -258,7 +259,7 @@ convert_enums!( Include(span: Span, path: String) i -> egglog::ast::Command::Include(i.span.clone().into(), (&i.path).into()), egglog::ast::Command::Include(span, p) => Include { span: span.into(), path: p.to_string() }; - Constructor(span: Span, name: String, schema: Schema, cost: Option, unextractable: bool) + Constructor(span: Span, name: String, schema: Schema, cost: Option, unextractable: bool) c -> egglog::ast::Command::Constructor { span: c.span.clone().into(), name: (&c.name).into(), @@ -325,7 +326,7 @@ convert_enums!( egglog::ast::Subdatatypes::NewSort(name, args) => NewSort { name: name.to_string(), args: args.iter().map(|e| e.into()).collect() } }; egglog::ExtractReport: "{:?}" => ExtractReport { - Best(termdag: TermDag, cost: usize, term: Term) + Best(termdag: TermDag, cost: DefaultCost, term: Term) b -> egglog::ExtractReport::Best { termdag: (&b.termdag).into(), cost: b.cost, @@ -384,7 +385,7 @@ convert_struct!( span: Span, name: String, types: Vec, - cost: Option = None + cost: Option = None ) v -> egglog::ast::Variant {span: v.span.clone().into(), name: (&v.name).into(), types: v.types.iter().map(|v| v.into()).collect(), cost: v.cost}, v -> Variant {span: v.span.clone().into(), name: v.name.to_string(), types: v.types.iter().map(|v| v.to_string()).collect(), cost: v.cost}; diff --git a/src/egraph.rs b/src/egraph.rs index 8a97fe24..9776c8a0 100644 --- a/src/egraph.rs +++ b/src/egraph.rs @@ -5,8 +5,8 @@ use crate::error::{EggResult, WrappedError}; use crate::py_object_sort::PyObjectSort; use crate::serialize::SerializedEGraph; -use egglog::prelude::add_leaf_sort; -use egglog::{span, SerializeConfig}; +use egglog::prelude::add_base_sort; +use egglog::{SerializeConfig, span}; use log::info; use pyo3::prelude::*; use std::path::PathBuf; @@ -38,7 +38,7 @@ impl EGraph { egraph.fact_directory = fact_directory; egraph.seminaive = seminaive; if let Some(py_object_sort) = py_object_sort { - add_leaf_sort(&mut egraph, py_object_sort, span!()).unwrap(); + add_base_sort(&mut egraph, py_object_sort, span!()).unwrap(); } Self { egraph, diff --git a/src/lib.rs b/src/lib.rs index 0f6acb98..439b01b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,11 +6,19 @@ mod serialize; mod termdag; mod utils; +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use pyo3::prelude::*; /// Bindings for egglog rust library #[pymodule] fn bindings(m: &Bound<'_, PyModule>) -> PyResult<()> { + rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build_global() + .unwrap(); + pyo3_log::init(); m.add_class::()?; From dbaf91a24b4acf7f246bdcb95a6c05bd150977f8 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Tue, 29 Jul 2025 16:05:28 +0100 Subject: [PATCH 2/4] Try updating gcc to fix minimalloc ppc build --- .github/workflows/version.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 2fda944c..770c28e1 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -59,6 +59,7 @@ jobs: needs: [bump] if: ${{ always() }} steps: + - run: sudo apt install -y gcc-X g++-X - uses: actions/checkout@v4 if: ${{ needs.bump.result == 'success' }} with: From 5addb60aa39c2f85ce645e148fa380fa539cae6c Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Tue, 29 Jul 2025 16:13:06 +0100 Subject: [PATCH 3/4] fix version --- .github/workflows/version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 770c28e1..ff91ea7a 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -59,7 +59,7 @@ jobs: needs: [bump] if: ${{ always() }} steps: - - run: sudo apt install -y gcc-X g++-X + - run: sudo apt install -y gcc g++ - uses: actions/checkout@v4 if: ${{ needs.bump.result == 'success' }} with: From 4a43cd8558074689d5b3368a1b36b711b2235fe6 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Tue, 29 Jul 2025 16:21:55 +0100 Subject: [PATCH 4/4] remove mimimalloc because of issue installing on ppc --- .github/workflows/version.yml | 1 - Cargo.lock | 35 ----------------------------------- Cargo.toml | 1 - src/lib.rs | 3 --- 4 files changed, 40 deletions(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index ff91ea7a..2fda944c 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -59,7 +59,6 @@ jobs: needs: [bump] if: ${{ always() }} steps: - - run: sudo apt install -y gcc g++ - uses: actions/checkout@v4 if: ${{ needs.bump.result == 'success' }} with: diff --git a/Cargo.lock b/Cargo.lock index 25d2a401..6a498714 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,15 +124,6 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" -[[package]] -name = "cc" -version = "1.2.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" -dependencies = [ - "shlex", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -422,7 +413,6 @@ dependencies = [ "egraph-serialize", "lalrpop-util", "log", - "mimalloc", "ordered-float", "pyo3", "pyo3-log", @@ -707,16 +697,6 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libmimalloc-sys" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -754,15 +734,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mimalloc" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "num" version = "0.4.3" @@ -1232,12 +1203,6 @@ dependencies = [ "digest", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "sized-chunks" version = "0.6.5" diff --git a/Cargo.toml b/Cargo.toml index 9ec631e4..fc4a8d94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ lalrpop-util = { version = "0.22", features = ["lexer"] } ordered-float = "3.7.0" uuid = { version = "1.17.0", features = ["v4"] } rayon = "1.10.0" -mimalloc = "0.1.46" # Use patched version of egglog in experimental [patch.'https://github.com/egraphs-good/egglog'] diff --git a/src/lib.rs b/src/lib.rs index 439b01b2..a34daad2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,6 @@ mod serialize; mod termdag; mod utils; -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - use pyo3::prelude::*; /// Bindings for egglog rust library