Merge pull request #15 from JuliaAI/tweak-faulty-test

Relax flawed test

Author: ablaom

test/parameters.jl

@@ -92,11 +92,10 @@
         ranges = [r1, (r2, Uniform), (r3, d3), r4]
         state = PSO.initialize(rng, ranges, n)
         PSO.retrieve!(rng, state)
-        @test state.parameters == (
-            ["a", "a", "c"],
-            [553, 250, 375],
-            [3.9372495283243105, 3.6569395920512977, 3.6354556967115146],
-            [-0.8067647f0, 0.4209916f0, 0.6736019f0]
-        )
+        params =  state.parameters
+        @test params[1] == ["a", "a", "c"]
+        @test params[2] ≈  [553, 250, 375]
+        @test params[3] ≈  [3.9372495283243105, 3.6569395920512977, 3.6354556967115146]
+        @test params[4] ≈ [-0.8067647f0, 0.4209916f0, 0.6736019f0]
     end
-end
+end

test/strategies/adaptive.jl

@@ -75,7 +75,10 @@ end
     @test c2 ≥ 1.8 # increase social
 end
 
-for acceleration in (CPU1(), CPUProcesses(), CPUThreads())
+const losses = []
+const modes = [CPU1(), CPUProcesses(), CPUThreads()]
+
+for acceleration in modes
     @testset "EvoTree Tuning with AdaptiveParticleSwarm and $(typeof(acceleration))" begin
         rng = StableRNG(123)
         features = rand(rng, 10_000) .* 5 .- 2
@@ -115,9 +118,14 @@ for acceleration in (CPU1(), CPUProcesses(), CPUThreads())
         fit!(mach, verbosity=0)
         rep = report(mach)
         best_loss = rep.best_history_entry.measurement[1]
+        push!(losses, best_loss)
+
+        # There is no reason to expect PSO to be better than
+        # RandomSearch, but they should give similar results, say within 10%:
 
-        # Compare with random search result with the same settings
-        @test best_loss < baseline_best_loss ||
-              isapprox(best_loss, baseline_best_loss; atol=1e-3)
+        @test abs(best_loss/baseline_best_loss - 1) < 0.1
     end
-end
+end
+
+println("Adaptive PSO losses (see Issue #14):")
+(; modes=modes, losses=losses) |> MLJBase.pretty