Skip to content

Commit 7e18ec5

Browse files
general-kroll-4-lifegeneral-kroll-4-life
committed
- Restored benchmarks.
1 parent 2348215 commit 7e18ec5

File tree

3 files changed

+162
-111
lines changed

3 files changed

+162
-111
lines changed

‎.github/workflows/build.yml‎

Lines changed: 106 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -308,59 +308,59 @@ jobs:
308308
if: success()
309309
run: python cicd/python/build.py --verbose --test
310310

311-
# - name: Benchmark
312-
# if: success()
313-
# run: |
314-
# outFile="cicd/log/current-bench-nocache.txt"
315-
# baselineRefFile="cicd/ref/bench/baseline-go-bench.log"
316-
# thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log"
317-
# go test -run='^$' -bench . -benchtime=100x -count=6 \
318-
# --tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \
319-
# ./... | tee ${outFile}
320-
# # shellcheck disable=SC2181
321-
# if [ "$?" = "0" ]; then
322-
# echo "Benchmarking run completed successfully"
323-
# else
324-
# echo "Benchmarking run failed"
325-
# exit 1
326-
# fi
327-
# echo ""
328-
# echo "##### Raw benchstat on current run #####"
329-
# echo ""
330-
# benchstat ${outFile} | tee cicd/log/raw-benchstat.txt
331-
# echo ""
332-
# echo "##### Comparing to baseline #####"
333-
# echo ""
334-
# benchstat -row .name -table .config -ignore goos,goarch,cpu ${baselineRefFile} ${outFile} | tee cicd/log/comparison-benchstat.txt
335-
# echo ""
336-
# echo "##### Comparing to max threshold #####"
337-
# echo ""
338-
# benchstat -row .name -table .config -ignore goos,goarch,cpu ${thresholdRefFile} ${outFile} | tee cicd/log/threshold-comparison-benchstat.txt
339-
# # shellcheck disable=SC2002,SC2062
340-
# comparisons=$( cat cicd/log/threshold-comparison-benchstat.txt \
341-
# | sed 's/.*\([+-][0-9][0-9]\.[0-9][0-9]*[%]\).*/\1/' \
342-
# | grep [+-][0-9][0-9]*\.[0-9][0-9]*[%] \
343-
# )
344-
# echo ""
345-
# echo "##### Comparisons #####"
346-
# echo ""
347-
# echo "${comparisons}"
348-
# # shellcheck disable=SC2062
349-
# nonNegativeComparisons=$( echo "${comparisons}" \
350-
# | grep -v [-].* \
351-
# || true
352-
# )
353-
# echo "completed comparison logic"
354-
# if [ -z "${nonNegativeComparisons}" ]; then
355-
# echo "All max threshold comparisons are negative: this is acceptable"
356-
# else
357-
# echo "Some max threshold comparisons are positive or zero: this is unacceptable"
358-
# echo ""
359-
# echo "##### Non-negative comparisons #####"
360-
# echo "${nonNegativeComparisons}"
361-
# echo ""
362-
# exit 1
363-
# fi
311+
- name: Benchmark
312+
if: success()
313+
run: |
314+
outFile="cicd/log/current-bench-nocache.txt"
315+
baselineRefFile="cicd/ref/bench/baseline-go-bench.log"
316+
thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log"
317+
go test -run='^$' -bench . -benchtime=100x -count=6 \
318+
--tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \
319+
./... | tee ${outFile}
320+
# shellcheck disable=SC2181
321+
if [ "$?" = "0" ]; then
322+
echo "Benchmarking run completed successfully"
323+
else
324+
echo "Benchmarking run failed"
325+
exit 1
326+
fi
327+
echo ""
328+
echo "##### Raw benchstat on current run #####"
329+
echo ""
330+
benchstat ${outFile} | tee cicd/log/raw-benchstat.txt
331+
echo ""
332+
echo "##### Comparing to baseline #####"
333+
echo ""
334+
benchstat -row .name -table .config -ignore goos,goarch,cpu ${baselineRefFile} ${outFile} | tee cicd/log/comparison-benchstat.txt
335+
echo ""
336+
echo "##### Comparing to max threshold #####"
337+
echo ""
338+
benchstat -row .name -table .config -ignore goos,goarch,cpu ${thresholdRefFile} ${outFile} | tee cicd/log/threshold-comparison-benchstat.txt
339+
# shellcheck disable=SC2002,SC2062
340+
comparisons=$( cat cicd/log/threshold-comparison-benchstat.txt \
341+
| sed 's/.*\([+-][0-9][0-9]\.[0-9][0-9]*[%]\).*/\1/' \
342+
| grep [+-][0-9][0-9]*\.[0-9][0-9]*[%] \
343+
)
344+
echo ""
345+
echo "##### Comparisons #####"
346+
echo ""
347+
echo "${comparisons}"
348+
# shellcheck disable=SC2062
349+
nonNegativeComparisons=$( echo "${comparisons}" \
350+
| grep -v [-].* \
351+
|| true
352+
)
353+
echo "completed comparison logic"
354+
if [ -z "${nonNegativeComparisons}" ]; then
355+
echo "All max threshold comparisons are negative: this is acceptable"
356+
else
357+
echo "Some max threshold comparisons are positive or zero: this is unacceptable"
358+
echo ""
359+
echo "##### Non-negative comparisons #####"
360+
echo "${nonNegativeComparisons}"
361+
echo ""
362+
exit 1
363+
fi
364364
365365
- name: Upload Artifact
366366
uses: actions/[email protected]
@@ -705,59 +705,59 @@ jobs:
705705
if: success()
706706
run: python cicd/python/build.py --verbose --test
707707

708-
# - name: Benchmark
709-
# if: success()
710-
# run: |
711-
# outFile="cicd/log/current-bench-nocache.txt"
712-
# baselineRefFile="cicd/ref/bench/baseline-go-bench.log"
713-
# thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log"
714-
# go test -run='^$' -bench . -benchtime=100x -count=6 \
715-
# --tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \
716-
# ./... | tee ${outFile}
717-
# # shellcheck disable=SC2181
718-
# if [ "$?" = "0" ]; then
719-
# echo "Benchmarking run completed successfully"
720-
# else
721-
# echo "Benchmarking run failed"
722-
# exit 1
723-
# fi
724-
# echo ""
725-
# echo "##### Raw benchstat on current run #####"
726-
# echo ""
727-
# benchstat ${outFile} | tee cicd/log/raw-benchstat.txt
728-
# echo ""
729-
# echo "##### Comparing to baseline #####"
730-
# echo ""
731-
# benchstat -row .name -table .config -ignore goos,goarch,cpu ${baselineRefFile} ${outFile} | tee cicd/log/comparison-benchstat.txt
732-
# echo ""
733-
# echo "##### Comparing to max threshold #####"
734-
# echo ""
735-
# benchstat -row .name -table .config -ignore goos,goarch,cpu ${thresholdRefFile} ${outFile} | tee cicd/log/threshold-comparison-benchstat.txt
736-
# # shellcheck disable=SC2002,SC2062
737-
# comparisons=$( cat cicd/log/threshold-comparison-benchstat.txt \
738-
# | sed 's/.*\([+-][0-9][0-9]\.[0-9][0-9]*[%]\).*/\1/' \
739-
# | grep [+-][0-9][0-9]*\.[0-9][0-9]*[%] \
740-
# )
741-
# echo ""
742-
# echo "##### Comparisons #####"
743-
# echo ""
744-
# echo "${comparisons}"
745-
# # shellcheck disable=SC2062
746-
# nonNegativeComparisons=$( echo "${comparisons}" \
747-
# | grep -v [-].* \
748-
# || true
749-
# )
750-
# echo "completed comparison logic"
751-
# if [ -z "${nonNegativeComparisons}" ]; then
752-
# echo "All max threshold comparisons are negative: this is acceptable"
753-
# else
754-
# echo "Some max threshold comparisons are positive or zero: this is unacceptable"
755-
# echo ""
756-
# echo "##### Non-negative comparisons #####"
757-
# echo "${nonNegativeComparisons}"
758-
# echo ""
759-
# exit 1
760-
# fi
708+
- name: Benchmark
709+
if: success()
710+
run: |
711+
outFile="cicd/log/current-bench-nocache.txt"
712+
baselineRefFile="cicd/ref/bench/baseline-go-bench.log"
713+
thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log"
714+
go test -run='^$' -bench . -benchtime=100x -count=6 \
715+
--tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \
716+
./... | tee ${outFile}
717+
# shellcheck disable=SC2181
718+
if [ "$?" = "0" ]; then
719+
echo "Benchmarking run completed successfully"
720+
else
721+
echo "Benchmarking run failed"
722+
exit 1
723+
fi
724+
echo ""
725+
echo "##### Raw benchstat on current run #####"
726+
echo ""
727+
benchstat ${outFile} | tee cicd/log/raw-benchstat.txt
728+
echo ""
729+
echo "##### Comparing to baseline #####"
730+
echo ""
731+
benchstat -row .name -table .config -ignore goos,goarch,cpu ${baselineRefFile} ${outFile} | tee cicd/log/comparison-benchstat.txt
732+
echo ""
733+
echo "##### Comparing to max threshold #####"
734+
echo ""
735+
benchstat -row .name -table .config -ignore goos,goarch,cpu ${thresholdRefFile} ${outFile} | tee cicd/log/threshold-comparison-benchstat.txt
736+
# shellcheck disable=SC2002,SC2062
737+
comparisons=$( cat cicd/log/threshold-comparison-benchstat.txt \
738+
| sed 's/.*\([+-][0-9][0-9]\.[0-9][0-9]*[%]\).*/\1/' \
739+
| grep [+-][0-9][0-9]*\.[0-9][0-9]*[%] \
740+
)
741+
echo ""
742+
echo "##### Comparisons #####"
743+
echo ""
744+
echo "${comparisons}"
745+
# shellcheck disable=SC2062
746+
nonNegativeComparisons=$( echo "${comparisons}" \
747+
| grep -v [-].* \
748+
|| true
749+
)
750+
echo "completed comparison logic"
751+
if [ -z "${nonNegativeComparisons}" ]; then
752+
echo "All max threshold comparisons are negative: this is acceptable"
753+
else
754+
echo "Some max threshold comparisons are positive or zero: this is unacceptable"
755+
echo ""
756+
echo "##### Non-negative comparisons #####"
757+
echo "${nonNegativeComparisons}"
758+
echo ""
759+
exit 1
760+
fi
761761
762762
- name: Create certificates for robot tests
763763
run: |

‎internal/stackql/driver/driver_bench_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func BenchmarkSelectGoogleComputeInstanceDriver(b *testing.B) {
3838
testhttpapi.StartServer(b, exp)
3939
provider.DummyAuth = true
4040

41-
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
41+
inputBundle, err := stackqltestutil.BuildBenchInputBundle(*runtimeCtx)
4242
if err != nil {
4343
b.Fatalf("Test failed: %v", err)
4444
}
@@ -87,7 +87,7 @@ func BenchmarkParallelProjectSelectGoogleComputeInstanceDriver(b *testing.B) {
8787
testhttpapi.StartServer(b, exp)
8888
provider.DummyAuth = true
8989

90-
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
90+
inputBundle, err := stackqltestutil.BuildBenchInputBundle(*runtimeCtx)
9191
if err != nil {
9292
b.Fatalf("Test failed: %v", err)
9393
}
@@ -267,7 +267,7 @@ func BenchmarkHighlyParallelProjectSelectGoogleComputeInstanceDriver(b *testing.
267267

268268
// runtimeCtx.ExecutionConcurrencyLimit = -1
269269

270-
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
270+
inputBundle, err := stackqltestutil.BuildBenchInputBundle(*runtimeCtx)
271271
if err != nil {
272272
b.Fatalf("Test failed: %v", err)
273273
}
@@ -475,7 +475,7 @@ func BenchmarkLoosenedHighlyParallelProjectSelectGoogleComputeInstanceDriver(b *
475475

476476
runtimeCtx.ExecutionConcurrencyLimit = 30
477477

478-
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
478+
inputBundle, err := stackqltestutil.BuildBenchInputBundle(*runtimeCtx)
479479
if err != nil {
480480
b.Fatalf("Test failed: %v", err)
481481
}
@@ -683,7 +683,7 @@ func BenchmarkUnlimitedHighlyParallelProjectSelectGoogleComputeInstanceDriver(b
683683

684684
runtimeCtx.ExecutionConcurrencyLimit = -1
685685

686-
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
686+
inputBundle, err := stackqltestutil.BuildBenchInputBundle(*runtimeCtx)
687687
if err != nil {
688688
b.Fatalf("Test failed: %v", err)
689689
}

‎internal/test/stackqltestutil/locator.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package stackqltestutil
33

44
import (
55
"fmt"
6+
"io"
67
"os"
78

89
"github.com/stackql/any-sdk/pkg/dto"
@@ -102,3 +103,53 @@ func BuildInputBundle(runtimeCtx dto.RuntimeCtx) (bundle.Bundle, error) {
102103
}
103104
return inputBundle, nil
104105
}
106+
107+
func BuildBenchInputBundle(runtimeCtx dto.RuntimeCtx) (bundle.Bundle, error) {
108+
inputBundle, err := entryutil.BuildInputBundle(runtimeCtx)
109+
if err != nil {
110+
return nil, err
111+
}
112+
googleRootDiscoveryBytes, err := getBytesFromLocalPath("test/db/google._root_.json")
113+
if err != nil {
114+
return nil, err
115+
}
116+
googleComputeDiscoveryBytes, err := getBytesFromLocalPath("test/db/google.compute.json")
117+
if err != nil {
118+
return nil, err
119+
}
120+
googleContainerDiscoveryBytes, err := getBytesFromLocalPath("test/db/google.container.json")
121+
if err != nil {
122+
return nil, err
123+
}
124+
googleCloudResourceManagerDiscoveryBytes, err := getBytesFromLocalPath("test/db/google.cloudresourcemanager.json")
125+
if err != nil {
126+
return nil, err
127+
}
128+
googleBQDiscoveryBytes, err := getBytesFromLocalPath("test/db/google.bigquery.json")
129+
if err != nil {
130+
return nil, err
131+
}
132+
sqlEng := inputBundle.GetSQLEngine()
133+
sqlEng.Exec(`INSERT INTO "__iql__.cache.key_val"(k, v) VALUES(?, ?)`, "https://www.googleapis.com/discovery/v1/apis", googleRootDiscoveryBytes)
134+
if err != nil {
135+
return nil, err
136+
}
137+
sqlEng.Exec(`INSERT INTO "__iql__.cache.key_val"(k, v) VALUES(?, ?)`, "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest", googleComputeDiscoveryBytes)
138+
if err != nil {
139+
return nil, err
140+
}
141+
sqlEng.Exec(`INSERT INTO "__iql__.cache.key_val"(k, v) VALUES(?, ?)`, "https://container.googleapis.com/$discovery/rest?version=v1", googleContainerDiscoveryBytes)
142+
if err != nil {
143+
return nil, err
144+
}
145+
sqlEng.Exec(`INSERT INTO "__iql__.cache.key_val"(k, v) VALUES(?, ?)`, "https://cloudresourcemanager.googleapis.com/$discovery/rest?version=v3", googleCloudResourceManagerDiscoveryBytes)
146+
if err != nil {
147+
return nil, err
148+
}
149+
sqlEng.Exec(`INSERT INTO "__iql__.cache.key_val"(k, v) VALUES(?, ?)`, "https://bigquery.googleapis.com/$discovery/rest?version=v2", googleBQDiscoveryBytes)
150+
if err != nil {
151+
return nil, err
152+
}
153+
inputBundle = inputBundle.WithStdErr(io.Discard).WithStdOut(io.Discard)
154+
return inputBundle, nil
155+
}

0 commit comments

Comments
 (0)