diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9095dd1f..4ca8feb9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,12 +24,14 @@ jobs: - gcc-version: 9 cmake-version: 3.14.7 upload: true - runtime-arg: 4 + runtime-arg: Linux-gcc-9 + memory-arg: Linux-gcc-9 # Latest supported gcc and cmake versions - gcc-version: 12 cmake-version: latest upload: false - runtime-arg: 5 + runtime-arg: Linux-gcc-12 + memory-arg: Linux-gcc-12 steps: - name: Checkout @@ -81,6 +83,11 @@ jobs: chmod +x test_scripts/verify_runtime.sh ./test_scripts/verify_runtime.sh "${{ matrix.runtime-arg }}" + - name: Run verify_memory.sh + run: | + chmod +x test_scripts/verify_memory.sh + ./test_scripts/verify_memory.sh "${{ matrix.memory-arg }}" + build-linux-aarch64: name: Linux aarch64 runs-on: ubuntu-22.04-arm @@ -125,7 +132,12 @@ jobs: - name: Run verify_runtimes.sh run: | chmod +x test_scripts/verify_runtime.sh - ./test_scripts/verify_runtime.sh 6 + ./test_scripts/verify_runtime.sh Linux-aarch64 + + - name: Run verify_memory.sh + run: | + chmod +x test_scripts/verify_memory.sh + ./test_scripts/verify_memory.sh Linux-aarch64 compile-linux-universal: name: Linux Universal @@ -210,7 +222,12 @@ jobs: - name: Run verify_runtimes.sh run: | chmod +x test_scripts/verify_runtime.sh - ./test_scripts/verify_runtime.sh 7 + ./test_scripts/verify_runtime.sh mac-os-x86-64 + + - name: Run verify_memory.sh + run: | + chmod +x test_scripts/verify_memory.sh + ./test_scripts/verify_memory.sh mac-os-x86-64 build-macos-arm: name: Mac OS ARM64 @@ -258,7 +275,12 @@ jobs: - name: Run verify_runtimes.sh run: | chmod +x test_scripts/verify_runtime.sh - ./test_scripts/verify_runtime.sh 8 + ./test_scripts/verify_runtime.sh mac-os-arm64 + + - name: Run verify_memory.sh + run: | + chmod +x test_scripts/verify_memory.sh + ./test_scripts/verify_memory.sh mac-os-arm64 compile-mac-universal: name: Mac OS Universal @@ -371,3 +393,7 @@ jobs: shell: pwsh run: | ./test_scripts/verify_runtime.ps1 + + - name: Run verify_memory.ps1 + run: | + ./test_scripts/verify_memory.ps1 \ No newline at end of file diff --git a/test_scripts/test_data/expected_memory.tsv b/test_scripts/test_data/expected_memory.tsv new file mode 100644 index 00000000..04dbdebd --- /dev/null +++ b/test_scripts/test_data/expected_memory.tsv @@ -0,0 +1,26 @@ +Command diff-threshold Linux-gcc-9 Linux-gcc-12 Linux-aarch64 mac-os-x86-64 mac-os-arm64 windows-x86 +build/iqtree3 -s test_scripts/test_data/turtle.fa -B 1000 -T 1 -seed 73073 3 18.98 18.89 17.73 13.53 15.53 20.09 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -seed 73073 3 19.04 19.05 17.79 13.18 13.97 20.16 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle.merge -seed 73073 3 19.15 19.08 17.99 13.54 15.52 17.65 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.merge.best_scheme.nex -z test_scripts/test_data/turtle.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle.test -seed 73073 -T 1 3 27.80 27.77 26.76 22.50 23.45 28.15 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m GTR+F+I+R3+T -te test_scripts/test_data/turtle.trees -T 1 --prefix test_scripts/test_data/turtle.mix -seed 73073 3 9.79 10.05 8.92 4.39 5.44 9.45 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex.best_scheme.nex -z test_scripts/test_data/turtle.trees -n 0 -wpl --prefix test_scripts/test_data/turtle.wpl -seed 73073 -T 1 3 9.27 9.30 8.18 3.61 4.75 9.43 +build/iqtree3 -s test_scripts/test_data/turtle.fa -S test_scripts/test_data/turtle.nex --prefix test_scripts/test_data/turtle.loci -T 1 -seed 73073 3 10.43 10.55 8.79 5.66 6.59 10.3 +build/iqtree3 -t test_scripts/test_data/turtle.nex.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed 73073 -T 1 3 6.30 6.55 5.92 2.29 2.75 6.57 +build/iqtree3 -t test_scripts/test_data/turtle.fa.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed 73073 -T 1 3 6.30 6.55 5.92 2.27 3.00 6.59 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX{GTR+FO,GTR+FO} --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.link -seed 73073 -T 1 3 9.17 9.05 7.92 3.47 4.80 9.2 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO} --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.jc.link -seed 73073 -T 1 3 9.17 8.84 7.92 3.74 4.49 9.36 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle.nex.constr -T 1 -seed 73073 3 10.57 10.45 8.80 5.86 6.78 10.43 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle.nex.constr2 -T 1 -seed 73073 3 18.82 18.70 17.55 13.29 14.00 19.74 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX+MF --prefix test_scripts/test_data/turtle.mixfinder -T 1 -seed 73073 3 14.55 15.50 11.68 9.21 9.28 20.19 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -B 1000 -T 1 -seed 73073 3 25.57 25.66 24.47 21.26 15.77 26.65 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -seed 73073 3 76.22 67.05 26.73 23.80 16.39 28.9 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle_aa.merge -seed 73073 3 41.34 31.46 26.11 22.30 16.56 28.35 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.merge.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle_aa.test -seed 73073 -T 1 3 33.53 33.46 32.43 31.32 30.30 33.82 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -n 0 -wpl --prefix test_scripts/test_data/turtle_aa.wpl -seed 73073 -T 1 3 18.26 18.32 16.95 14.17 12.81 14.99 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -S test_scripts/test_data/turtle_aa.nex --prefix test_scripts/test_data/turtle_aa.loci -T 1 -seed 73073 3 16.49 16.11 13.53 11.07 9.69 14.96 +build/iqtree3 -t test_scripts/test_data/turtle_aa.nex.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed 73073 -T 1 3 6.30 6.55 5.92 2.38 3.06 6.74 +build/iqtree3 -t test_scripts/test_data/turtle_aa.fasta.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed 73073 -T 1 3 6.29 6.55 5.93 2.29 3.09 6.64 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m MIX{LG+F,WAG+F} --prefix test_scripts/test_data/turtle_aa.mix -seed 73073 -T 1 3 11.79 11.80 10.67 6.39 6.78 11.94 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle_aa.nex.constr -T 1 -seed 73073 3 59.79 60.66 18.28 18.67 10.61 16.95 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle_aa.nex.constr2 -T 1 -seed 73073 3 63.79 70.88 26.53 23.55 16.72 26.39 diff --git a/test_scripts/test_data/expected_runtime.tsv b/test_scripts/test_data/expected_runtime.tsv new file mode 100644 index 00000000..d4aa8392 --- /dev/null +++ b/test_scripts/test_data/expected_runtime.tsv @@ -0,0 +1,26 @@ +Command diff-threshold Linux-gcc-9 Linux-gcc-12 Linux-aarch64 mac-os-x86-64 mac-os-arm64 windows-x86 +build/iqtree3 -s test_scripts/test_data/turtle.fa -B 1000 -T 1 -seed 73073 2 3.24 3.32 5.03 3.62 2.96 4.55 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -seed 73073 2 3.46 3.59 4.32 3.63 2.74 4.06 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle.merge -seed 73073 2 3.35 3.49 5.01 4.14 3.00 3.86 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.merge.best_scheme.nex -z test_scripts/test_data/turtle.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle.test -seed 73073 -T 1 2 1.34 1.31 0.74 0.85 0.78 3.26 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m GTR+F+I+R3+T -te test_scripts/test_data/turtle.trees -T 1 --prefix test_scripts/test_data/turtle.mix -seed 73073 2 2.43 2.36 3.76 2.29 2.52 2.85 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex.best_scheme.nex -z test_scripts/test_data/turtle.trees -n 0 -wpl --prefix test_scripts/test_data/turtle.wpl -seed 73073 -T 1 2 0.05 0.05 0.05 0.13 0.04 0.62 +build/iqtree3 -s test_scripts/test_data/turtle.fa -S test_scripts/test_data/turtle.nex --prefix test_scripts/test_data/turtle.loci -T 1 -seed 73073 2 3.64 3.82 4.88 3.90 2.94 3.86 +build/iqtree3 -t test_scripts/test_data/turtle.nex.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed 73073 -T 1 1 0.01 0.01 0.01 0.02 0.01 0.4 +build/iqtree3 -t test_scripts/test_data/turtle.fa.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed 73073 -T 1 1 0.01 0.01 0.01 0.02 0.01 0.41 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX{GTR+FO,GTR+FO} --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.link -seed 73073 -T 1 2 0.79 0.82 1.23 0.69 0.64 0.4 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO} --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.jc.link -seed 73073 -T 1 2 0.70 0.74 1.09 0.82 0.71 0.41 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle.nex.constr -T 1 -seed 73073 2 3.05 3.17 3.83 3.62 2.22 3.47 +build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle.nex.constr2 -T 1 -seed 73073 2 3.50 3.58 4.25 3.95 2.49 4.27 +build/iqtree3 -s test_scripts/test_data/turtle.fa -m MIX+MF --prefix test_scripts/test_data/turtle.mixfinder -T 1 -seed 73073 3 9.15 8.47 14.31 9.81 8.20 0.42 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -B 1000 -T 1 -seed 73073 12 62.06 65.05 100.61 35.56 40.99 35.76 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -seed 73073 12 71.33 72.66 116.11 44.03 46.82 41.86 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle_aa.merge -seed 73073 12 65.65 67.02 106.90 39.94 40.45 37.61 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.merge.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle_aa.test -seed 73073 -T 1 2 2.66 2.61 3.17 1.43 1.61 3.26 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -n 0 -wpl --prefix test_scripts/test_data/turtle_aa.wpl -seed 73073 -T 1 2 1.80 1.83 2.92 1.08 1.27 1.62 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -S test_scripts/test_data/turtle_aa.nex --prefix test_scripts/test_data/turtle_aa.loci -T 1 -seed 73073 12 55.46 56.38 89.44 34.67 35.94 32.52 +build/iqtree3 -t test_scripts/test_data/turtle_aa.nex.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed 73073 -T 1 1 0.01 0.01 0.01 0.02 0.02 0.4 +build/iqtree3 -t test_scripts/test_data/turtle_aa.fasta.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed 73073 -T 1 1 0.01 0.01 0.01 0.02 0.01 0.41 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m MIX{LG+F,WAG+F} --prefix test_scripts/test_data/turtle_aa.mix -seed 73073 -T 1 3 13.04 13.33 21.36 7.23 8.97 7.53 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle_aa.nex.constr -T 1 -seed 73073 12 61.22 61.69 99.07 37.95 39.54 35.39 +build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle_aa.nex.constr2 -T 1 -seed 73073 12 82.86 85.39 135.76 49.92 53.91 48.37 diff --git a/test_scripts/test_data/expected_runtimes.tsv b/test_scripts/test_data/expected_runtimes.tsv deleted file mode 100644 index 527e905c..00000000 --- a/test_scripts/test_data/expected_runtimes.tsv +++ /dev/null @@ -1,22 +0,0 @@ -#file name key-words diff-threshold Linux-gcc-9 Linux-gcc-12 Linux-aarch64 mac-os-x86-64 mac-os-arm64 windows-x86 -turtle.fa.iqtree Total CPU time used 2 3.128169 3.299343 4.909145 4.93771 3.595236 3.21875 -turtle.loci.iqtree Total CPU time used 2 3.376 3.6 4.561 5.737 2.786 3.359 -turtle.merge.iqtree Total CPU time used 2 3.22134 3.336272 4.881305 4.074864 3.063053 3.34375 -turtle.mix.iqtree Total CPU time used 1 2.288 2.437 3.653 2.043 2.607 2.391 -turtle.mix.jc.link.iqtree Total CPU time used 1 0.678048 0.716485 1.05925 0.79052 0.798578 0.8125 -turtle.mix.link.iqtree Total CPU time used 1 0.765474 0.829717 1.19517 0.856055 0.868658 0.90625 -turtle.mixfinder.iqtree Total CPU time used 3 9.34105 9.95267 14.5544 9.57637 8.78048 8.40625 -turtle.nex.constr.iqtree Total CPU time used 3 2.74033 2.88714 3.45122 4.30755 2.83421 2.85938 -turtle.nex.constr2.iqtree Total CPU time used 3 3.172122 3.296328 3.914713 4.247386 3.261848 3.484375 -turtle.nex.iqtree Total CPU time used 3 3.139167 3.257494 4.000374 4.824423 3.217744 3.578125 -turtle.test.iqtree Total CPU time used 1 0.235 0.242 0.349 0.192 0.225 0.234 -turtle.wpl.iqtree Total CPU time used 1 0.037859 0.036415 0.043837 0.041274 0.030318 0.015625 -turtle_aa.fasta.iqtree Total CPU time used 12 61.60878 61.761949 100.241028 44.144877 44.523198 34.46875 -turtle_aa.loci.iqtree Total CPU time used 12 55.21 56.31 89.16 32.3 45.828247 31.66 -turtle_aa.merge.iqtree Total CPU time used 12 65.057715 65.34153 106.422985 48.539932 47.526024 36.890625 -turtle_aa.nex.constr.iqtree Total CPU time used 12 60.3527 60.7689 98.7859 47.122 38.9924 34.4219 -turtle_aa.nex.constr2.iqtree Total CPU time used 12 82.473306 82.646098 135.013106 54.514873 58.861398 47.40625 -turtle_aa.nex.iqtree Total CPU time used 12 70.691887 71.079425 115.466386 54.95159 52.16108 40.890625 -turtle_aa.test.iqtree Total CPU time used 1 1.69 1.81 2.72 0.862 0.979 0.922 -turtle_aa.wpl.iqtree Total CPU time used 3 1.6987 1.701 2.7838 0.89948 1.0239 0.96875 -turtle_aa.mix.iqtree Total CPU time used 3 12.8918 13.1371 21.3145 6.91331 9.35718 7.09375 \ No newline at end of file diff --git a/test_scripts/test_iqtree.ps1 b/test_scripts/test_iqtree.ps1 index 593c484f..ac3b5e16 100755 --- a/test_scripts/test_iqtree.ps1 +++ b/test_scripts/test_iqtree.ps1 @@ -1,71 +1,116 @@ $SEED=73073 +$LOGFILE = "time_log.tsv" +"Command`tRealTime(s)`tPeakMemory(MB)" | Out-File -FilePath $LOGFILE -Encoding utf8 + +function Measure-IQTree { + param ( + [string]$CommandLine + ) + + Write-Host "`n===== RUNNING: $CommandLine =====" + + $startTime = Get-Date + + # Split command and arguments + $exe, $args = $CommandLine -split '\s+', 2 + $tempOut = [System.IO.Path]::GetTempFileName() + + # Start the process with output redirection + $proc = Start-Process -FilePath $exe -ArgumentList $args ` + -RedirectStandardOutput $tempOut ` + -NoNewWindow -PassThru + + $procId = $proc.Id + $peakMemory = 0 + + while (-not $proc.HasExited) { + Start-Sleep -Milliseconds 200 + try { + $currentMem = (Get-Process -Id $procId -ErrorAction Stop).WorkingSet64 / 1MB + if ($currentMem -gt $peakMemory) { + $peakMemory = $currentMem + } + } catch { + break + } + } + + $endTime = Get-Date + $elapsed = [math]::Round(($endTime - $startTime).TotalSeconds, 2) + + # Show output + Get-Content $tempOut + + # Log timing and memory + "$CommandLine`t$elapsed`t$([math]::Round($peakMemory, 2))" | Out-File -FilePath $LOGFILE -Append -Encoding utf8 + + # Cleanup + Remove-Item $tempOut +} + + + # ./build/iqtree3 -s test_scripts/test_data/small.fa -p test_scripts/test_data/small.nex -m "MFP+MERGE" -T 1 -seed $SEED -./build/iqtree3 -s test_scripts/test_data/turtle.fa -B 1000 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -B 1000 -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle.merge -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle.merge -seed $SEED" Get-Content test_scripts/test_data/turtle.fa.treefile, test_scripts/test_data/turtle.nex.treefile | Set-Content test_scripts/test_data/turtle.trees -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.merge.best_scheme.nex -z test_scripts/test_data/turtle.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle.test -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.merge.best_scheme.nex -z test_scripts/test_data/turtle.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle.test -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -m GTR+F+I+R3+T -te test_scripts/test_data/turtle.trees -T 1 --prefix test_scripts/test_data/turtle.mix -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -m GTR+F+I+R3+T -te test_scripts/test_data/turtle.trees -T 1 --prefix test_scripts/test_data/turtle.mix -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex.best_scheme.nex -z test_scripts/test_data/turtle.trees -n 0 -wpl --prefix test_scripts/test_data/turtle.wpl -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex.best_scheme.nex -z test_scripts/test_data/turtle.trees -n 0 -wpl --prefix test_scripts/test_data/turtle.wpl -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -S test_scripts/test_data/turtle.nex --prefix test_scripts/test_data/turtle.loci -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -S test_scripts/test_data/turtle.nex --prefix test_scripts/test_data/turtle.loci -T 1 -seed $SEED" -./build/iqtree3 -t test_scripts/test_data/turtle.nex.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -t test_scripts/test_data/turtle.nex.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1" -./build/iqtree3 -t test_scripts/test_data/turtle.fa.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -t test_scripts/test_data/turtle.fa.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1" # link-exchange-rates model -./build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX{GTR+FO,GTR+FO}" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.link -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -m `"MIX{GTR+FO,GTR+FO}`" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.link -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.jc.link -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -m `"MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}`" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.jc.link -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle.nex.constr -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle.nex.constr -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle.nex.constr2 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle.nex.constr2 -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX+MF" --prefix test_scripts/test_data/turtle.mixfinder -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX+MF" --prefix test_scripts/test_data/turtle.mixfinder -T 1 -seed $SEED" ## amino acid test cases # the data set is a subset of the turtle data set echo "Running amino acid test cases..." -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -B 1000 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -B 1000 -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -seed $SEED" -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle_aa.merge -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle_aa.merge -seed $SEED" Get-Content test_scripts/test_data/turtle_aa.fasta.treefile, test_scripts/test_data/turtle_aa.nex.treefile | Set-Content test_scripts/test_data/turtle_aa.trees -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.merge.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle_aa.test -seed $SEED -T 1 - -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m GTR+F+I+R3+T -te test_scripts/test_data/turtle_aa.trees -T 1 --prefix test_scripts/test_data/turtle_aa.mix -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.merge.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle_aa.test -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -n 0 -wpl --prefix test_scripts/test_data/turtle_aa.wpl -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex.best_scheme.nex -z test_scripts/test_data/turtle_aa.trees -n 0 -wpl --prefix test_scripts/test_data/turtle_aa.wpl -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -S test_scripts/test_data/turtle_aa.nex --prefix test_scripts/test_data/turtle_aa.loci -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -S test_scripts/test_data/turtle_aa.nex --prefix test_scripts/test_data/turtle_aa.loci -T 1 -seed $SEED" -./build/iqtree3 -t test_scripts/test_data/turtle_aa.nex.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -t test_scripts/test_data/turtle_aa.nex.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed $SEED -T 1" -./build/iqtree3 -t test_scripts/test_data/turtle_aa.fasta.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed $SEED -T 1 +Measure-IQTree "./build/iqtree3 -t test_scripts/test_data/turtle_aa.fasta.treefile --gcf test_scripts/test_data/turtle_aa.loci.treefile -s test_scripts/test_data/turtle_aa.fasta --scf 100 -seed $SEED -T 1" # link-exchange-rates model -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m "MIX{LG+F,WAG+F}" --prefix test_scripts/test_data/turtle_aa.mix -seed $SEED -T 1 - -#./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m "MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}" --link-exchange-rates --prefix test_scripts/test_data/turtle_aa.mix.jc.link -seed $SEED -T 1 - -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle_aa.nex.constr -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m `"MIX{LG+F,WAG+F}`" --prefix test_scripts/test_data/turtle_aa.mix -seed $SEED -T 1" -./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle_aa.nex.constr2 -T 1 -seed $SEED +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle_aa.nex.constr -T 1 -seed $SEED" -#./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -m "MIX+MF" --prefix test_scripts/test_data/turtle_aa.mixfinder -T 1 -seed $SEED # mixture finder is not supported for amino acid data +Measure-IQTree "./build/iqtree3 -s test_scripts/test_data/turtle_aa.fasta -p test_scripts/test_data/turtle_aa.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle_aa.nex.constr2 -T 1 -seed $SEED" diff --git a/test_scripts/test_iqtree.sh b/test_scripts/test_iqtree.sh index 61cc7a8a..157f4197 100755 --- a/test_scripts/test_iqtree.sh +++ b/test_scripts/test_iqtree.sh @@ -2,76 +2,102 @@ SEED=73073 -#build/iqtree3 -s test_scripts/test_data/small.fa -p test_scripts/test_data/small.nex -m "MFP+MERGE" -T 1 -seed $SEED +LOGFILE="time_log.tsv" +WD=test_scripts/test_data +BUILD_DIR="build" + +# Initialize TSV file with header +echo -e "Command\tRealTime(s)\tPeakMemory(MB)" > "$LOGFILE" + +run_timed() { + local CMD="$*" + echo -e "\n================ RUNNING =================" + echo "$CMD" + echo "==========================================" + + local OS=$(uname) + local REAL USER SYS PEAK_MEM MEM_MB + + if [[ "$OS" == "Darwin" ]]; then + # macOS + /usr/bin/time -l "$@" 2> tmp_time.txt + REAL=$(grep "real" tmp_time.txt | awk '{print $1}') + USER=$(grep "user" tmp_time.txt | awk '{print $1}') + SYS=$(grep "sys" tmp_time.txt | awk '{print $1}') + PEAK_MEM=$(grep "peak memory footprint" tmp_time.txt | awk '{print $1}') + MEM_MB=$(awk "BEGIN {printf \"%.2f\", $PEAK_MEM / (1024 * 1024)}") + else + # Linux + /usr/bin/time -f "%e %U %S %M" "$@" 2> tmp_time.txt + read REAL USER SYS MEM_KB < tmp_time.txt + MEM_MB=$(awk "BEGIN {printf \"%.2f\", $MEM_KB / 1024}") + fi + + # Append to log + echo -e "$CMD\t$REAL\t$MEM_MB" >> "$LOGFILE" -build/iqtree3 -s test_scripts/test_data/turtle.fa -B 1000 -T 1 -seed $SEED + rm -f tmp_time.txt +} -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -B 1000 -T 1 -seed $SEED -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix test_scripts/test_data/turtle.merge -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.nex -B 1000 -T 1 -seed $SEED -cat test_scripts/test_data/turtle.fa.treefile test_scripts/test_data/turtle.nex.treefile > test_scripts/test_data/turtle.trees -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.merge.best_scheme.nex -z test_scripts/test_data/turtle.trees -zb 10000 -au -n 0 --prefix test_scripts/test_data/turtle.test -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.nex -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix ${WD}/turtle.merge -seed $SEED -build/iqtree3 -s test_scripts/test_data/turtle.fa -m GTR+F+I+R3+T -te test_scripts/test_data/turtle.trees -T 1 --prefix test_scripts/test_data/turtle.mix -seed $SEED +cat ${WD}/turtle.fa.treefile ${WD}/turtle.nex.treefile > ${WD}/turtle.trees +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.merge.best_scheme.nex -z ${WD}/turtle.trees -zb 10000 -au -n 0 --prefix ${WD}/turtle.test -seed $SEED -T 1 -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex.best_scheme.nex -z test_scripts/test_data/turtle.trees -n 0 -wpl --prefix test_scripts/test_data/turtle.wpl -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -m GTR+F+I+R3+T -te ${WD}/turtle.trees -T 1 --prefix ${WD}/turtle.mix -seed $SEED -build/iqtree3 -s test_scripts/test_data/turtle.fa -S test_scripts/test_data/turtle.nex --prefix test_scripts/test_data/turtle.loci -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.nex.best_scheme.nex -z ${WD}/turtle.trees -n 0 -wpl --prefix ${WD}/turtle.wpl -seed $SEED -T 1 -build/iqtree3 -t test_scripts/test_data/turtle.nex.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -S ${WD}/turtle.nex --prefix ${WD}/turtle.loci -T 1 -seed $SEED -build/iqtree3 -t test_scripts/test_data/turtle.fa.treefile --gcf test_scripts/test_data/turtle.loci.treefile -s test_scripts/test_data/turtle.fa --scf 100 -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -t ${WD}/turtle.nex.treefile --gcf ${WD}/turtle.loci.treefile -s ${WD}/turtle.fa --scf 100 -seed $SEED -T 1 + +run_timed ${BUILD_DIR}/iqtree3 -t ${WD}/turtle.fa.treefile --gcf ${WD}/turtle.loci.treefile -s ${WD}/turtle.fa --scf 100 -seed $SEED -T 1 # link-exchange-rates model -build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX{GTR+FO,GTR+FO}" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.link -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -m "MIX{GTR+FO,GTR+FO}" --link-exchange-rates --prefix ${WD}/turtle.mix.link -seed $SEED -T 1 -build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}" --link-exchange-rates --prefix test_scripts/test_data/turtle.mix.jc.link -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -m "MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}" --link-exchange-rates --prefix ${WD}/turtle.mix.jc.link -seed $SEED -T 1 -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree --prefix test_scripts/test_data/turtle.nex.constr -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.nex -g ${WD}/turtle.constr.tree --prefix ${WD}/turtle.nex.constr -T 1 -seed $SEED -build/iqtree3 -s test_scripts/test_data/turtle.fa -p test_scripts/test_data/turtle.nex -g test_scripts/test_data/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix test_scripts/test_data/turtle.nex.constr2 -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -p ${WD}/turtle.nex -g ${WD}/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix ${WD}/turtle.nex.constr2 -T 1 -seed $SEED -build/iqtree3 -s test_scripts/test_data/turtle.fa -m "MIX+MF" --prefix test_scripts/test_data/turtle.mixfinder -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s ${WD}/turtle.fa -m "MIX+MF" --prefix ${WD}/turtle.mixfinder -T 1 -seed $SEED ## amino acid test cases # the data set is a subset of the turtle data set echo "Running amino acid test cases..." -WD=test_scripts/test_data -BUILD_DIR="build" AA_FASTA=${WD}/turtle_aa.fasta AA_NEX=${WD}/turtle_aa.nex AA_prefix=${WD}/turtle_aa -${BUILD_DIR}/iqtree3 -s $AA_FASTA -B 1000 -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -B 1000 -T 1 -seed $SEED -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -B 1000 -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -B 1000 -T 1 -seed $SEED -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix ${WD}/turtle_aa.merge -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -B 1000 -T 1 -m MFP+MERGE -rcluster 10 --prefix ${WD}/turtle_aa.merge -seed $SEED cat $AA_FASTA.treefile $AA_NEX.treefile > ${WD}/turtle_aa.trees -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p ${WD}/turtle_aa.merge.best_scheme.nex -z ${WD}/turtle_aa.trees -zb 10000 -au -n 0 --prefix ${WD}/turtle_aa.test -seed $SEED -T 1 - -${BUILD_DIR}/iqtree3 -s $AA_FASTA -m GTR+F+I+R3+T -te ${WD}/turtle_aa.trees -T 1 --prefix ${WD}/turtle_aa.mix -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p ${WD}/turtle_aa.merge.best_scheme.nex -z ${WD}/turtle_aa.trees -zb 10000 -au -n 0 --prefix ${WD}/turtle_aa.test -seed $SEED -T 1 -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX.best_scheme.nex -z ${WD}/turtle_aa.trees -n 0 -wpl --prefix ${WD}/turtle_aa.wpl -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX.best_scheme.nex -z ${WD}/turtle_aa.trees -n 0 -wpl --prefix ${WD}/turtle_aa.wpl -seed $SEED -T 1 -${BUILD_DIR}/iqtree3 -s $AA_FASTA -S $AA_NEX --prefix ${WD}/turtle_aa.loci -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -S $AA_NEX --prefix ${WD}/turtle_aa.loci -T 1 -seed $SEED -${BUILD_DIR}/iqtree3 -t $AA_NEX.treefile --gcf ${WD}/turtle_aa.loci.treefile -s $AA_FASTA --scf 100 -seed $SEED -T 1 - -${BUILD_DIR}/iqtree3 -t $AA_FASTA.treefile --gcf ${WD}/turtle_aa.loci.treefile -s $AA_FASTA --scf 100 -seed $SEED -T 1 - -# link-exchange-rates model +run_timed ${BUILD_DIR}/iqtree3 -t $AA_NEX.treefile --gcf ${WD}/turtle_aa.loci.treefile -s $AA_FASTA --scf 100 -seed $SEED -T 1 -${BUILD_DIR}/iqtree3 -s $AA_FASTA -m "MIX{LG+F,WAG+F}" --prefix ${WD}/turtle_aa.mix -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -t $AA_FASTA.treefile --gcf ${WD}/turtle_aa.loci.treefile -s $AA_FASTA --scf 100 -seed $SEED -T 1 -#${BUILD_DIR}/iqtree3 -s $AA_FASTA -m "MIX{GTR{1,1,1,1,1,1}+FO,GTR{1,1,1,1,1,1}+FO}" --link-exchange-rates --prefix ${WD}/turtle_aa.mix.jc.link -seed $SEED -T 1 +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -m "MIX{LG+F,WAG+F}" --prefix ${WD}/turtle_aa.mix -seed $SEED -T 1 -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -g ${WD}/turtle.constr.tree --prefix $AA_NEX.constr -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -g ${WD}/turtle.constr.tree --prefix $AA_NEX.constr -T 1 -seed $SEED -${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -g ${WD}/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix $AA_NEX.constr2 -T 1 -seed $SEED +run_timed ${BUILD_DIR}/iqtree3 -s $AA_FASTA -p $AA_NEX -g ${WD}/turtle.constr.tree2 -B 1000 -alrt 1000 --prefix $AA_NEX.constr2 -T 1 -seed $SEED -#${BUILD_DIR}/iqtree3 -s $AA_FASTA -m "MIX+MF" --prefix ${WD}/turtle_aa.mixfinder -T 1 -seed $SEED # mixture finder is not supported for amino acid data diff --git a/test_scripts/verify_memory.ps1 b/test_scripts/verify_memory.ps1 new file mode 100644 index 00000000..d830604f --- /dev/null +++ b/test_scripts/verify_memory.ps1 @@ -0,0 +1,77 @@ +param ( + [string] $ExpectedColumnName = "windows-x86" +) + +$WD = "test_scripts/test_data" +$expectedFile = Join-Path $WD "expected_memory.tsv" +$reportedFile = "time_log.tsv" +$selectedColumnsFile = Join-Path $WD "selected_columns.tsv" +$reportedColumnFile = "$env:TEMP\reported_column.tsv" +$finalFile = Join-Path $WD "combined_with_reported.tsv" + +# Get the header and find the index of the expected column name +$header = Get-Content $expectedFile -TotalCount 1 +$columns = $header -split "`t" +$columnIndex = $columns.IndexOf($ExpectedColumnName) + +if ($columnIndex -lt 0) { + Write-Error "Column '$ExpectedColumnName' not found in $expectedFile" + exit 1 +} + +# Adjust to 0-based indexing for arrays +$expectedLines = Get-Content $expectedFile | Select-Object -Skip 1 +$selectedColumns = foreach ($line in $expectedLines) { + $parts = $line -split "`t" + "$($parts[0])`t$($parts[1])`t$($parts[$columnIndex])" +} +$selectedColumns | Set-Content $selectedColumnsFile + +# Read reported column from time_log.tsv (column 3 = RealTime, 4 = Memory) +$reportedLines = Get-Content $reportedFile | Select-Object -Skip 1 +$reportedColumn = foreach ($line in $reportedLines) { + $parts = $line -split "`t" + $parts[2] # column 3 = memory (0-based index) +} +$reportedColumn | Set-Content $reportedColumnFile + +# Combine both files +$combinedLines = @() +for ($i = 0; $i -lt $selectedColumns.Count; $i++) { + $combinedLines += "$($selectedColumns[$i])`t$($reportedColumn[$i])" +} +$combinedLines | Set-Content $finalFile + +# Now compare +$failCount = 0 +$finalLines = Get-Content $finalFile + +foreach ($line in $finalLines) { + $parts = $line -split "`t" + $command = $parts[0] + $threshold = [double]$parts[1] + $expected = [double]$parts[2] + $reported = [double]$parts[3] + + $allowed = $expected + $threshold + $exceededBy = $reported - $expected + + if ($reported -gt $allowed) { + Write-Host "❌ $command exceeded the allowed usage." + Write-Host "Expected: $expected MB, Threshold: $threshold MB, Reported: $reported MB" + $failCount++ + } else { + Write-Host "✅ $command passed the check." + Write-Host "Expected: $expected MB, Threshold: $threshold MB, Reported: $reported MB" + } +} + +Write-Host "" + +if ($failCount -eq 0) { + Write-Host "✅ All memory checks passed." + exit 0 +} else { + Write-Host "❌ $failCount checks failed." + exit 1 +} diff --git a/test_scripts/verify_memory.sh b/test_scripts/verify_memory.sh new file mode 100644 index 00000000..ae23f9b9 --- /dev/null +++ b/test_scripts/verify_memory.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# expect the first argument to be the column name : contain what platform(OS) is running the script +expected_column="$1" + +WD="test_scripts/test_data" +input_file="${WD}/expected_memory.tsv" +reported_file="time_log.tsv" +selected_columns_file="${WD}/selected_columns.tsv" + +# Get the column index (1-based) of the expected column name +col_index=$(head -1 "$input_file" | tr '\t' '\n' | awk -v col="$expected_column" '{if ($0 == col) print NR}') + +# Check if column was found +if [[ -z "$col_index" ]]; then + echo "Column '$expected_column' not found in $input_file" + exit 1 +fi + +cut -f1,2,"$col_index" "$input_file" > "${selected_columns_file}" +final_file="${WD}/combined_with_reported.tsv" + +# assuming the reported file and expected file have the same order of commands +temp_reported_column=$(mktemp) +cut -f3 "$reported_file" > "$temp_reported_column" +paste "$selected_columns_file" "$temp_reported_column" > "$final_file" +rm -f "$temp_reported_column" + + +fail_count=0 + +# Skip header +while IFS=$'\t' read -r command threshold expected reported; do + allowed=$(echo "$expected + $threshold" | bc -l) + is_exceed=$(echo "$reported > $allowed" | bc -l) + + if [ "$is_exceed" = "1" ]; then + diff=$(echo "$reported - $expected" | bc -l) + echo "❌ $command exceeded the allowed memory usage." + echo "Expected: $expected MB, Threshold: $threshold MB, Reported: $reported MB, Difference: $diff MB" + ((fail_count++)) + else + echo "✅ $command passed the memory check." + diff=$(echo "$reported - $expected" | bc -l) + echo "Expected: $expected MB, Threshold: $threshold MB, Reported: $reported MB, Difference: $diff MB" + fi +done < <(tail -n +2 "$final_file") + +if [ "$fail_count" -eq 0 ]; then + echo "✅ All memory checks passed." +else + echo "❌ $fail_count checks failed." + exit 1 +fi diff --git a/test_scripts/verify_runtime.ps1 b/test_scripts/verify_runtime.ps1 index 188fe902..ea97ea1b 100755 --- a/test_scripts/verify_runtime.ps1 +++ b/test_scripts/verify_runtime.ps1 @@ -1,63 +1,77 @@ -param( - [int] $ExpectedColumn = 8, # 1-based index of the threshold column - [string]$input_file = "test_scripts/test_data/expected_runtimes.tsv" +param ( + [string] $ExpectedColumnName = "windows-x86" ) -$fail_count = 0 -$line_num = 0 +$WD = "test_scripts/test_data" +$expectedFile = Join-Path $WD "expected_runtime.tsv" +$reportedFile = "time_log.tsv" +$selectedColumnsFile = Join-Path $WD "selected_columns.tsv" +$reportedColumnFile = "$env:TEMP\reported_column.tsv" +$finalFile = Join-Path $WD "combined_with_reported.tsv" -Get-Content $input_file | ForEach-Object { - $line = $_.Trim() - $line_num++ +# Get the header and find the index of the expected column name +$header = Get-Content $expectedFile -TotalCount 1 +$columns = $header -split "`t" +$columnIndex = $columns.IndexOf($ExpectedColumnName) - if ($line_num -eq 1 -or $line -eq "") { - return # skip header or empty line - } - - $columns = $line -split "`t" - if ($columns.Count -lt 4) { - Write-Host "Skipping malformed line: $line" - return - } +if ($columnIndex -lt 0) { + Write-Error "Column '$ExpectedColumnName' not found in $expectedFile" + exit 1 +} - $iqtree_file = Join-Path "test_scripts/test_data" $columns[0] - $field_name = $columns[1] - $expected_value = [double]$columns[$ExpectedColumn] - $threshold = [double]$columns[2] +# Adjust to 0-based indexing for arrays +$expectedLines = Get-Content $expectedFile | Select-Object -Skip 1 +$selectedColumns = foreach ($line in $expectedLines) { + $parts = $line -split "`t" + "$($parts[0])`t$($parts[1])`t$($parts[$columnIndex])" +} +$selectedColumns | Set-Content $selectedColumnsFile - if (-not (Test-Path $iqtree_file)) { - Write-Host "File not found: ${iqtree_file}" - return - } +# Read reported column from time_log.tsv (column 3 = RealTime, 4 = Memory) +$reportedLines = Get-Content $reportedFile | Select-Object -Skip 1 +$reportedColumn = foreach ($line in $reportedLines) { + $parts = $line -split "`t" + $parts[1] # column 2 = runtime (0-based index) +} +$reportedColumn | Set-Content $reportedColumnFile - $actual_line = Select-String -Path $iqtree_file -Pattern ([regex]::Escape($field_name)) - if (-not $actual_line) { - Write-Host "Field not found in ${iqtree_file}: ${field_name}" - return - } +# Combine both files +$combinedLines = @() +for ($i = 0; $i -lt $selectedColumns.Count; $i++) { + $combinedLines += "$($selectedColumns[$i])`t$($reportedColumn[$i])" +} +$combinedLines | Set-Content $finalFile - $match = [regex]::Match($actual_line.Line, '[-+]?\d+(\.\d+)?([eE][-+]?\d+)?') +# Now compare +$failCount = 0 +$finalLines = Get-Content $finalFile - if (-not $match.Success) { - Write-Host "No numeric value found in line: $($actual_line.Line)" - return - } +foreach ($line in $finalLines) { + $parts = $line -split "`t" + $command = $parts[0] + $threshold = [double]$parts[1] + $expected = [double]$parts[2] + $reported = [double]$parts[3] - $actual_value = [double]$match.Value - $highest_value = $expected_value + $threshold + $allowed = $expected + $threshold + $exceededBy = $reported - $expected - if ($actual_value -le $highest_value) { - Write-Host "PASS: ${iqtree_file} -- Expected: ${expected_value}, Reported: ${actual_value}, Threshold: ${threshold}" + if ($reported -gt $allowed) { + Write-Host "❌ $command exceeded the allowed usage." + Write-Host "Expected: $expected S, Threshold: $threshold S, Reported: $reported S" + $failCount++ } else { - Write-Host "FAIL: ${iqtree_file} -- Expected: ${expected_value}, Reported: ${actual_value}, Threshold: ${threshold}" - $fail_count++ + Write-Host "✅ $command passed the check." + Write-Host "Expected: $expected S, Threshold: $threshold S, Reported: $reported S" } } Write-Host "" -if ($fail_count -eq 0) { + +if ($failCount -eq 0) { Write-Host "✅ All runtime checks passed." + exit 0 } else { - Write-Host "❌ ${fail_count} checks failed." + Write-Host "❌ $failCount checks failed." exit 1 } diff --git a/test_scripts/verify_runtime.sh b/test_scripts/verify_runtime.sh index f20f0790..f42513bd 100644 --- a/test_scripts/verify_runtime.sh +++ b/test_scripts/verify_runtime.sh @@ -1,57 +1,48 @@ #!/bin/bash +# expect the first argument to be the column name : contain what platform(OS) is running the script expected_column="$1" WD="test_scripts/test_data" -input_file="${WD}/expected_runtimes.tsv" -selected_colums_file="${WD}/selected_columns.tsv" -cut -f1,2,3,"$expected_column" "$input_file" > "${selected_colums_file}" +input_file="${WD}/expected_runtime.tsv" +reported_file="time_log.tsv" +selected_columns_file="${WD}/selected_columns.tsv" -fail_count=0 -line_num=0 - -while IFS=$'\t' read -r iqtree_file field_name threshold expected_value || [ -n "$iqtree_file" ]; do - ((line_num++)) +# Get the column index (1-based) of the expected column name +col_index=$(head -1 "$input_file" | tr '\t' '\n' | awk -v col="$expected_column" '{if ($0 == col) print NR}') - # Skip header (first line) - if [ "$line_num" -eq 1 ]; then - continue - fi +# Check if column was found +if [[ -z "$col_index" ]]; then + echo "Column '$expected_column' not found in $input_file" + exit 1 +fi - iqtree_file="${WD}/${iqtree_file}" +cut -f1,2,"$col_index" "$input_file" > "${selected_columns_file}" +final_file="${WD}/combined_with_reported.tsv" - if [ ! -f "$iqtree_file" ]; then - echo "File not found: $iqtree_file" - continue - fi +# assuming the reported file and expected file have the same order of commands +cut -f2 "$reported_file" > /tmp/reported_column.tsv +paste "$selected_columns_file" /tmp/reported_column.tsv > "$final_file" - # Look for the line containing the field name - report_line=$(grep -F "$field_name" "$iqtree_file") - if [ -z "$report_line" ]; then - echo "Field not found in $iqtree_file: $field_name" - continue - fi - # Extract the first numeric value from the matched line - report_value=$(echo "$report_line" | grep -Eo '[-+]?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?' | head -n1) - if [ -z "$report_value" ]; then - echo "No numeric value found in line: $report_line" - continue - fi +fail_count=0 - # Compute report value less than the highest value - higest_value=$(echo "$expected_value + $threshold" | bc -l) - result=$(echo "$report_value <= $higest_value" | bc -l) +# Skip header +while IFS=$'\t' read -r command threshold expected reported; do + allowed=$(echo "$expected + $threshold" | bc -l) + is_exceed=$(echo "$reported > $allowed" | bc -l) - if [ "$result" -eq 1 ]; then - echo "PASS: $iqtree_file -- Expected: ${expected_value}, Reported: ${report_value}, Threshold: $threshold" - else - echo "FAIL: $iqtree_file ($field_name)" - echo " Expected: ${expected_value}, Reported: ${report_value}, Threshold: $threshold" + if [ "$is_exceed" = "1" ]; then + diff=$(echo "$reported - $expected" | bc -l) + echo "❌ $command exceeded the allowed runtime usage." + echo "Expected: $expected S, Threshold: $threshold S, Reported: $reported S, Difference: $diff S" ((fail_count++)) + else + echo "✅ $command passed the runtime check." + diff=$(echo "$reported - $expected" | bc -l) + echo "Expected: $expected S, Threshold: $threshold S, Reported: $reported S, Difference: $diff S" fi -done < "$selected_colums_file" +done < <(tail -n +2 "$final_file") -echo if [ "$fail_count" -eq 0 ]; then echo "✅ All runtime checks passed." else