es hyperneat support

.cache/v/cache/lastfailed

@@ -0,0 +1,9 @@
+{
+  "tests/test_adaptive_linear.py": true,
+  "tests/test_cppn.py": true,
+  "tests/test_maze.py": true,
+  "tests/test_recurrent.py": true,
+  "tests/test_strict_t_maze.py": true,
+  "tests/test_t_maze.py": true,
+  "tests/test_turning_t_maze.py": true
+}

examples/es-hyperneat/main.py

@@ -0,0 +1,93 @@
+import multiprocessing
+import os
+
+import click
+import neat
+import gym
+# import torch
+import numpy as np
+import tensorflow as tf
+
+from tf_neat import t_maze
+from tf_neat.activations import tanh_activation
+from tf_neat.adaptive_linear_net import AdaptiveLinearNet
+from tf_neat.multi_env_eval import MultiEnvEvaluator
+from tf_neat.neat_reporter import LogReporter
+from tf_neat.es_hyperneat import ESNetwork
+from tf_neat.substrate import Substrate
+from tf_neat.cppn import create_cppn
+
+
+max_env_steps = 200
+
+
+def make_env():
+    return gym.make("CartPole-v0")
+
+def make_net(genome, config, bs):
+    #start by setting up a substrate for this bad cartpole boi
+    params = {"initial_depth": 2,
+            "max_depth": 4,
+            "variance_threshold": 0.00013,
+            "band_threshold": 0.00013,
+            "iteration_level": 3,
+            "division_threshold": 0.00013,
+            "max_weight": 3.0,
+            "activation": "tanh"}
+    input_cords = []
+    output_cords = [(0.0, -1.0, 0.0)]
+    sign = 1
+    # we will use a 3 dimensional substrate, coords laid out here
+    for i in range(3):
+        input_cords.append((0.0 - i/10*sign, 1.0, 0.0))
+        sign *= -1
+    leaf_names = []
+    for i in range(3):
+        leaf_names.append('leaf_one_'+str(i))
+        leaf_names.append('leaf_two_'+str(i))
+
+    [cppn] = create_cppn(genome, config, leaf_names, ['cppn_out'])
+    net_builder = ESNetwork(Substrate(input_cords, output_cords), cppn, params)
+    net = net_builder.create_phenotype_network_nd('./genome_vis')
+    return net
+
+def activate_net(net, states):
+    outputs = net.activate(states).numpy()
+    return outputs[0] > 0.5
+
+
+@click.command()
+@click.option("--n_generations", type=int, default=100)
+def run(n_generations):
+    # Load the config file, which is assumed to live in
+    # the same directory as this script.
+    config_path = os.path.join(os.path.dirname(__file__), "neat.cfg")
+    config = neat.Config(
+        neat.DefaultGenome,
+        neat.DefaultReproduction,
+        neat.DefaultSpeciesSet,
+        neat.DefaultStagnation,
+        config_path,
+    )
+
+    evaluator = MultiEnvEvaluator(
+        make_net, activate_net, make_env=make_env, max_env_steps=max_env_steps
+    )
+
+    def eval_genomes(genomes, config):
+        for _, genome in genomes:
+            genome.fitness = evaluator.eval_genome(genome, config)
+
+    pop = neat.Population(config)
+    stats = neat.StatisticsReporter()
+    pop.add_reporter(stats)
+    reporter = neat.StdOutReporter(True)
+    pop.add_reporter(reporter)
+    #logger = LogReporter("neat.log", evaluator.eval_genome)
+    #pop.add_reporter(logger)
+
+    pop.run(eval_genomes, n_generations)
+
+
+if __name__ == "__main__":
+    run()  # pylint: disable=no-value-for-parameter

examples/es-hyperneat/neat.cfg

@@ -0,0 +1,61 @@
+# The `NEAT` section specifies parameters particular to the NEAT algorithm
+# or the experiment itself.  This is the only required section.
+[NEAT]
+fitness_criterion     = max
+fitness_threshold     = 200
+pop_size              = 250
+reset_on_extinction   = 0
+
+[DefaultGenome]
+num_inputs              = 6
+num_hidden              = 1
+num_outputs             = 1
+initial_connection      = partial_direct 0.5
+feed_forward            = True
+compatibility_disjoint_coefficient    = 1.0
+compatibility_weight_coefficient      = 0.6
+conn_add_prob           = 0.2
+conn_delete_prob        = 0.2
+node_add_prob           = 0.2
+node_delete_prob        = 0.2
+activation_default      = sigmoid
+activation_options      = sigmoid abs gauss sin identity
+activation_mutate_rate  = 0.0
+aggregation_default     = sum
+aggregation_options     = sum
+aggregation_mutate_rate = 0.0
+bias_init_mean          = 0.0
+bias_init_stdev         = 1.0
+bias_replace_rate       = 0.1
+bias_mutate_rate        = 0.7
+bias_mutate_power       = 0.5
+bias_max_value          = 30.0
+bias_min_value          = -30.0
+response_init_mean      = 1.0
+response_init_stdev     = 0.0
+response_replace_rate   = 0.0
+response_mutate_rate    = 0.0
+response_mutate_power   = 0.0
+response_max_value      = 30.0
+response_min_value      = -30.0
+
+weight_max_value        = 30
+weight_min_value        = -30
+weight_init_mean        = 0.0
+weight_init_stdev       = 1.0
+weight_mutate_rate      = 0.8
+weight_replace_rate     = 0.1
+weight_mutate_power     = 0.5
+enabled_default         = True
+enabled_mutate_rate     = 0.01
+
+[DefaultSpeciesSet]
+compatibility_threshold = 3.0
+
+[DefaultStagnation]
+species_fitness_func = max
+max_stagnation  = 20
+
+[DefaultReproduction]
+elitism            = 2
+survival_threshold = 0.2

tests/test_nd_get_coords.py

@@ -0,0 +1,26 @@
+import torch
+from tf_neat.cppn import get_nd_coord_inputs, get_coord_inputs, create_cppn
+
+def test_coords():
+    input_coords = [[-1.0, 0.0, 1.0], [0.0, 0.0, 1.0], [1.0, 0.0, 1.0], [-1.0, 0.0, 1.0]]
+    output_coords = [[-2.0, 0.0, 2.0], [0.0, 0.0, -2.0], [2.0, 0.0, 1.0]]
+
+    input_coords_2d = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0], [0.0, -1.0]]
+    output_coords_2d = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0]]
+
+    inputs = torch.tensor(
+        input_coords, dtype=torch.float32
+    )
+    outputs = torch.tensor(
+        output_coords, dtype=torch.float32
+    )
+    inputs_2 = torch.tensor(
+        input_coords_2d, dtype=torch.float32
+    )
+    outputs_2 = torch.tensor(
+        output_coords_2d, dtype=torch.float32
+    )
+    print(get_coord_inputs(inputs_2, outputs_2))
+    print(get_nd_coord_inputs(inputs, outputs))
+
+test_coords()

tf_neat/adaptive_example.py

@@ -0,0 +1,130 @@
+# Copyright (c) 2018 Uber Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+import multiprocessing
+import os
+
+import click
+import neat
+
+# import torch
+import numpy as np
+import tensorflow as tf
+
+import t_maze
+from activations import tanh_activation
+from adaptive_linear_net import AdaptiveLinearNet
+from multi_env_eval import MultiEnvEvaluator
+from neat_reporter import LogReporter
+
+# Activate eager TensorFlow execution
+tf.enable_eager_execution()
+print("Executing eagerly: ", tf.executing_eagerly())
+
+batch_size = 4
+DEBUG = False
+
+
+def make_net(genome, config, _batch_size):
+    input_coords = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0], [0.0, -1.0]]
+    output_coords = [[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0]]
+    return AdaptiveLinearNet.create(
+        genome,
+        config,
+        input_coords=input_coords,
+        output_coords=output_coords,
+        weight_threshold=0.4,
+        batch_size=batch_size,
+        activation=tanh_activation,
+        output_activation=tanh_activation,
+        device="cpu",
+    )
+
+
+def activate_net(net, states, debug=False, step_num=0):
+    if debug and step_num == 1:
+        print("\n" + "=" * 20 + " DEBUG " + "=" * 20)
+        print(net.delta_w_node)
+        print("W init: ", net.input_to_output[0])
+    outputs = net.activate(states).numpy()
+    if debug and (step_num - 1) % 100 == 0:
+        print("\nStep {}".format(step_num - 1))
+        print("Outputs: ", outputs[0])
+        print("Delta W: ", net.delta_w[0])
+        print("W: ", net.input_to_output[0])
+    return np.argmax(outputs, axis=1)
+
+
+@click.command()
+@click.option("--n_generations", type=int, default=10000)
+@click.option("--n_processes", type=int, default=1)
+def run(n_generations, n_processes):
+    # Load the config file, which is assumed to live in
+    # the same directory as this script.
+    config_path = os.path.join(os.path.dirname(__file__), "neat.cfg")
+    config = neat.Config(
+        neat.DefaultGenome,
+        neat.DefaultReproduction,
+        neat.DefaultSpeciesSet,
+        neat.DefaultStagnation,
+        config_path,
+    )
+
+    envs = [t_maze.TMazeEnv(init_reward_side=i, n_trials=100) for i in [1, 0, 1, 0]]
+
+    evaluator = MultiEnvEvaluator(
+        make_net, activate_net, envs=envs, batch_size=batch_size, max_env_steps=1000
+    )
+
+    if n_processes > 1:
+        pool = multiprocessing.Pool(processes=n_processes)
+
+        def eval_genomes(genomes, config):
+            fitnesses = pool.starmap(
+                evaluator.eval_genome, ((genome, config) for _, genome in genomes)
+            )
+            for (_, genome), fitness in zip(genomes, fitnesses):
+                genome.fitness = fitness
+
+    else:
+
+        def eval_genomes(genomes, config):
+            for i, (_, genome) in enumerate(genomes):
+                try:
+                    genome.fitness = evaluator.eval_genome(
+                        genome, config, debug=DEBUG and i % 100 == 0
+                    )
+                except Exception as e:
+                    print(genome)
+                    raise e
+
+    pop = neat.Population(config)
+    stats = neat.StatisticsReporter()
+    pop.add_reporter(stats)
+    reporter = neat.StdOutReporter(True)
+    pop.add_reporter(reporter)
+    logger = LogReporter("./logs/adaptive.json", evaluator.eval_genome)
+    pop.add_reporter(logger)
+
+    winner = pop.run(eval_genomes, n_generations)
+
+    print(winner)
+    final_performance = evaluator.eval_genome(winner, config)
+    print("Final performance: {}".format(final_performance))
+    generations = reporter.generation + 1
+    return generations
+
+
+if __name__ == "__main__":
+    run()  # pylint: disable=no-value-for-parameter

tf_neat/adaptive_linear_net.py

@@ -17,9 +17,9 @@
 
 import tensorflow as tf
 
-from .activations import identity_activation, tanh_activation
-from .cppn import clamp_weights_, create_cppn, get_coord_inputs
-from .helpers import expand
+from activations import identity_activation, tanh_activation
+from cppn import clamp_weights_, create_cppn, get_coord_inputs
+from helpers import expand
 
 
 class AdaptiveLinearNet:

tf_neat/cppn.py

@@ -97,14 +97,13 @@ def get_activs(self, shape):
             self.activs = self.activate(xs, shape)
         return self.activs
 
-    def __call__(self, **inputs):
+    def __call__(self, inputs={}):
         assert self.leaves is not None
         assert inputs
         shape = list(inputs.values())[0].shape
         self.reset()
         for name in self.leaves.keys():
-            assert (inputs[name].shape == shape), \
-                "Wrong activs shape for leaf {}, {} != {}".format(name, inputs[name].shape, shape)
+            assert inputs[name].shape == shape,"Wrong activs shape for leaf {}, {} != {}".format(name, inputs[name].shape, shape)
             self.leaves[name].set_activs(inputs[name])
         return self.get_activs(shape)
 
@@ -267,3 +266,16 @@ def get_coord_inputs(in_coords, out_coords, batch_size=None):
         y_in = expand(tf.expand_dims(in_coords[:, 1], 0), (n_out, n_in))
 
     return (x_out, y_out), (x_in, y_in)
+
+def get_nd_coord_inputs(in_coords, out_coords):
+    n_in = in_coords.shape[0]
+    n_out = out_coords.shape[0]
+
+    dims = in_coords.shape[1]
+
+    arrays = {}
+    for x in range(dims):
+        arrays[str(x) + "_out"] = expand(tf.expand_dims(out_coords[:, x], 1), (n_out, n_in))
+        arrays[str(x) + "_in"] = expand(tf.expand_dims(in_coords[:, x], 0), (n_out, n_in))
+
+    return arrays