diff --git a/Cargo.toml b/Cargo.toml index 590afdc..70b272b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ license = "MIT" [features] default = ["max-index"] -#crossover = ["genetic-rs/crossover"] +crossover = ["genetic-rs/crossover"] rayon = ["genetic-rs/rayon", "dep:rayon"] max-index = [] diff --git a/src/runnable.rs b/src/runnable.rs index efcd3b2..5bdb82d 100644 --- a/src/runnable.rs +++ b/src/runnable.rs @@ -82,7 +82,7 @@ impl NeuralNetwork { n.state.value += self.process_neuron(l) * w; } - n.sigmoid(); + n.activate(); n.state.value } @@ -112,7 +112,7 @@ impl NeuralNetwork { let mut nw = n.write().unwrap(); nw.state.value += val; - nw.sigmoid(); + nw.activate(); nw.state.value } @@ -240,6 +240,9 @@ pub struct Neuron { /// The current state of the neuron. pub state: NeuronState, + + /// The neuron's activation function + pub activation: ActivationFn, } impl Neuron { @@ -248,9 +251,9 @@ impl Neuron { self.state.value = self.bias; } - /// Applies the sigoid activation function to the state's current value. - pub fn sigmoid(&mut self) { - self.state.value = 1. / (1. + std::f32::consts::E.powf(-self.state.value)) + /// Applies the activation function to the neuron + pub fn activate(&mut self) { + self.state.value = (self.activation.func)(self.state.value); } } @@ -263,6 +266,7 @@ impl From<&NeuronTopology> for Neuron { value: value.bias, ..Default::default() }, + activation: value.activation.clone(), } } } diff --git a/src/topology.rs b/src/topology.rs index 2c9f09f..384de65 100644 --- a/src/topology.rs +++ b/src/topology.rs @@ -1,8 +1,27 @@ -use std::sync::{Arc, RwLock}; +use std::{ + collections::HashSet, + fmt, + sync::{Arc, RwLock}, +}; use genetic_rs::prelude::*; use rand::prelude::*; +/// Creates an [`ActivationFn`] object from a function +#[macro_export] +macro_rules! activation_fn { + ($F: path) => { + ActivationFn { + func: Arc::new($F), + name: String::from(stringify!($F)), + } + }; + + {$($F: path),*} => { + [$(activation_fn!($F)),*] + }; +} + /// A stateless neural network topology. /// This is the struct you want to use in your agent's inheritance. /// See [`NeuralNetwork::from`][crate::NeuralNetwork::from] for how to convert this to a runnable neural network. @@ -28,7 +47,13 @@ impl NeuralNetworkTopology { /// Creates a new [`NeuralNetworkTopology`]. pub fn new(mutation_rate: f32, mutation_passes: usize, rng: &mut impl Rng) -> Self { let input_layer: [Arc>; I] = (0..I) - .map(|_| Arc::new(RwLock::new(NeuronTopology::new(vec![], rng)))) + .map(|_| { + Arc::new(RwLock::new(NeuronTopology::new_with_activation( + vec![], + activation_fn!(linear_activation), + rng, + ))) + }) .collect::>() .try_into() .unwrap(); @@ -51,7 +76,11 @@ impl NeuralNetworkTopology { }) .collect(); - output_layer.push(Arc::new(RwLock::new(NeuronTopology::new(input, rng)))); + output_layer.push(Arc::new(RwLock::new(NeuronTopology::new_with_activation( + input, + activation_fn!(sigmoid), + rng, + )))); } let output_layer = output_layer.try_into().unwrap(); @@ -65,17 +94,61 @@ impl NeuralNetworkTopology { } } - fn is_connection_cyclic(&self, loc1: NeuronLocation, loc2: NeuronLocation) -> bool { - if loc1 == loc2 { + /// Creates a new connection between the neurons. + /// If the connection is cyclic, it does not add a connection and returns false. + /// Otherwise, it returns true. + pub fn add_connection( + &mut self, + from: NeuronLocation, + to: NeuronLocation, + weight: f32, + ) -> bool { + if self.is_connection_cyclic(from, to) { + return false; + } + + // Add the connection since it is not cyclic + self.get_neuron(to) + .write() + .unwrap() + .inputs + .push((from, weight)); + + true + } + + fn is_connection_cyclic(&self, from: NeuronLocation, to: NeuronLocation) -> bool { + if to.is_input() || from.is_output() { + return true; + } + + let mut visited = HashSet::new(); + self.dfs(from, to, &mut visited) + } + + // TODO rayon implementation + fn dfs( + &self, + current: NeuronLocation, + target: NeuronLocation, + visited: &mut HashSet, + ) -> bool { + if current == target { return true; } - for &(n, _w) in &self.get_neuron(loc1).read().unwrap().inputs { - if self.is_connection_cyclic(n, loc2) { + visited.insert(current); + + let n = self.get_neuron(current); + let nr = n.read().unwrap(); + + for &(input, _) in &nr.inputs { + if !visited.contains(&input) && self.dfs(input, target, visited) { return true; } } + visited.remove(¤t); false } @@ -96,11 +169,7 @@ impl NeuralNetworkTopology { let i = rng.gen_range(0..self.input_layer.len()); (self.input_layer[i].clone(), NeuronLocation::Input(i)) } - 1 => { - if self.hidden_layers.is_empty() { - return self.rand_neuron(rng); - } - + 1 if !self.hidden_layers.is_empty() => { let i = rng.gen_range(0..self.hidden_layers.len()); (self.hidden_layers[i].clone(), NeuronLocation::Hidden(i)) } @@ -110,6 +179,64 @@ impl NeuralNetworkTopology { } } } + + fn delete_neuron(&mut self, loc: NeuronLocation) -> NeuronTopology { + if !loc.is_hidden() { + panic!("Invalid neuron deletion"); + } + + let index = loc.unwrap(); + let neuron = Arc::into_inner(self.hidden_layers.remove(index)).unwrap(); + + for n in &self.hidden_layers { + let mut nw = n.write().unwrap(); + + nw.inputs = nw + .inputs + .iter() + .filter_map(|&(input_loc, w)| { + if !input_loc.is_hidden() { + return Some((input_loc, w)); + } + + if input_loc.unwrap() == index { + return None; + } + + if input_loc.unwrap() > index { + return Some((NeuronLocation::Hidden(input_loc.unwrap() - 1), w)); + } + + Some((input_loc, w)) + }) + .collect(); + } + + for n2 in &self.output_layer { + let mut nw = n2.write().unwrap(); + nw.inputs = nw + .inputs + .iter() + .filter_map(|&(input_loc, w)| { + if !input_loc.is_hidden() { + return Some((input_loc, w)); + } + + if input_loc.unwrap() == index { + return None; + } + + if input_loc.unwrap() > index { + return Some((NeuronLocation::Hidden(input_loc.unwrap() - 1), w)); + } + + Some((input_loc, w)) + }) + .collect(); + } + + neuron.into_inner().unwrap() + } } // need to do all this manually because Arcs are cringe @@ -163,27 +290,35 @@ impl RandomlyMutable for NeuralNetworkTopology() <= rate { // add a connection - let (mut n1, mut loc1) = self.rand_neuron(rng); + let (_, mut loc1) = self.rand_neuron(rng); + let (_, mut loc2) = self.rand_neuron(rng); - while n1.read().unwrap().inputs.is_empty() { - (n1, loc1) = self.rand_neuron(rng); + while loc1.is_output() || !self.add_connection(loc1, loc2, rng.gen::()) { + (_, loc1) = self.rand_neuron(rng); + (_, loc2) = self.rand_neuron(rng); } + } - let (mut n2, mut loc2) = self.rand_neuron(rng); + if rng.gen::() <= rate && !self.hidden_layers.is_empty() { + // remove a neuron + let (_, mut loc) = self.rand_neuron(rng); - while self.is_connection_cyclic(loc1, loc2) { - (n2, loc2) = self.rand_neuron(rng); + while !loc.is_hidden() { + (_, loc) = self.rand_neuron(rng); } - n2.write().unwrap().inputs.push((loc1, rng.gen())); + // delete the neuron + self.delete_neuron(loc); } if rng.gen::() <= rate { @@ -199,6 +334,34 @@ impl RandomlyMutable for NeuralNetworkTopology() <= rate { + // mutate bias + let (n, _) = self.rand_neuron(rng); + let mut n = n.write().unwrap(); + + n.bias += rng.gen_range(-1.0..1.0) * rate; + } + + if rng.gen::() <= rate && !self.hidden_layers.is_empty() { + // mutate activation function + let activations = activation_fn! { + sigmoid, + relu, + f32::tanh + }; + + let (mut n, mut loc) = self.rand_neuron(rng); + + while !loc.is_hidden() { + (n, loc) = self.rand_neuron(rng); + } + + let mut nw = n.write().unwrap(); + + // should probably not clone, but its not a huge efficiency issue anyways + nw.activation = activations[rng.gen_range(0..activations.len())].clone(); + } } } } @@ -211,12 +374,43 @@ impl DivisionReproduction for NeuralNetworkTopol } } +/* #[cfg(feature = "crossover")] impl CrossoverReproduction for NeuralNetworkTopology { fn crossover(&self, other: &Self, rng: &mut impl Rng) -> Self { todo!(); } } +*/ + +/// An activation function object that implements [`fmt::Debug`] and is [`Send`] +#[derive(Clone)] +pub struct ActivationFn { + /// The actual activation function. + pub func: Arc f32 + Send + Sync + 'static>, + name: String, +} + +impl fmt::Debug for ActivationFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{}", self.name) + } +} + +/// The sigmoid activation function. +pub fn sigmoid(n: f32) -> f32 { + 1. / (1. + std::f32::consts::E.powf(-n)) +} + +/// The ReLU activation function. +pub fn relu(n: f32) -> f32 { + n.max(0.) +} + +/// Activation function that does nothing. +pub fn linear_activation(n: f32) -> f32 { + n +} /// A stateless version of [`Neuron`][crate::Neuron]. #[derive(Debug, Clone)] @@ -226,22 +420,59 @@ pub struct NeuronTopology { /// The neuron's bias. pub bias: f32, + + /// The neuron's activation function. + pub activation: ActivationFn, } impl NeuronTopology { /// Creates a new neuron with the given input locations. pub fn new(inputs: Vec, rng: &mut impl Rng) -> Self { - let inputs = inputs.into_iter().map(|i| (i, rng.gen::())).collect(); + let activations = activation_fn! { + sigmoid, + relu, + f32::tanh + }; + + Self::new_with_activations(inputs, activations, rng) + } + + /// Takes a collection of activation functions and chooses a random one to use. + pub fn new_with_activations( + inputs: Vec, + activations: impl IntoIterator, + rng: &mut impl Rng, + ) -> Self { + let mut activations: Vec<_> = activations.into_iter().collect(); + + Self::new_with_activation( + inputs, + activations.remove(rng.gen_range(0..activations.len())), + rng, + ) + } + + /// Creates a neuron with the activation. + pub fn new_with_activation( + inputs: Vec, + activation: ActivationFn, + rng: &mut impl Rng, + ) -> Self { + let inputs = inputs + .into_iter() + .map(|i| (i, rng.gen_range(-1.0..1.0))) + .collect(); Self { inputs, bias: rng.gen(), + activation, } } } /// A pseudo-pointer of sorts used to make structural conversions very fast and easy to write. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Hash, Clone, Copy, Debug, Eq, PartialEq)] pub enum NeuronLocation { /// Points to a neuron in the input layer at contained index. Input(usize),