From b5c6dd493c106d937bf05603d31e5e1fc1ac3b39 Mon Sep 17 00:00:00 2001 From: "C.A.P. Linssen" Date: Thu, 9 Jan 2025 14:56:36 +0100 Subject: [PATCH] generate NESTML model code --- .../neuromodulated_stdp_synapse.nestml | 71 ++++++++----------- .../polebalancing.ipynb | 66 +++++++++++++---- 2 files changed, 82 insertions(+), 55 deletions(-) diff --git a/doc/tutorials/cart_pole_reinforcement_learning/neuromodulated_stdp_synapse.nestml b/doc/tutorials/cart_pole_reinforcement_learning/neuromodulated_stdp_synapse.nestml index 5ae1beef5..1421a2646 100644 --- a/doc/tutorials/cart_pole_reinforcement_learning/neuromodulated_stdp_synapse.nestml +++ b/doc/tutorials/cart_pole_reinforcement_learning/neuromodulated_stdp_synapse.nestml @@ -1,68 +1,55 @@ """ -model neuromodulated_stdp_synapse: +stdp - Synapse model for spike-timing dependent plasticity +######################################################### + +... +""" +model neuromodulated_stdp_synapse: state: - w real = 1. - n real = 0. # Neuromodulator concentration - c real = 0. # Eligibility trace - pre_tr real = 0. - post_tr real = 0. + w real = 1 # Synaptic weight + pre_trace real = 0. + post_trace real = 0. parameters: - d ms = 1 ms - tau_tr_pre ms = 20 ms # STDP time constant for weight changes caused by pre-before-post spike pairings. - tau_tr_post ms = 20 ms # STDP time constant for weight changes caused by post-before-pre spike pairings. - tau_c ms = 1000 ms # Time constant of eligibility trace - tau_n ms = 200 ms # Time constant of dopaminergic trace - b real = 0. # Dopaminergic baseline concentration - Wmax real = 200. # Maximal synaptic weight - Wmin real = 0. # Minimal synaptic weight - A_plus real = 1. # Multiplier applied to weight changes caused by pre-before-post spike pairings. If b (dopamine baseline concentration) is zero, then A_plus is simply the multiplier for facilitation (as in the stdp_synapse model). If b is not zero, then A_plus will be the multiplier for facilitation only if n - b is positive, where n is the instantenous dopamine concentration in the volume transmitter. If n - b is negative, A_plus will be the multiplier for depression. - A_minus real = 1.5 # Multiplier applied to weight changes caused by post-before-pre spike pairings. If b (dopamine baseline concentration) is zero, then A_minus is simply the multiplier for depression (as in the stdp_synapse model). If b is not zero, then A_minus will be the multiplier for depression only if n - b is positive, where n is the instantenous dopamine concentration in the volume transmitter. If n - b is negative, A_minus will be the multiplier for facilitation. - A_vt real = 1. # Multiplier applied to dopa spikes + n real = 0. # neuromodulator concentration between 0 and 1 + d ms = 1 ms # Synaptic transmission delay + lambda real = .01 + tau_tr_pre ms = 20 ms + tau_tr_post ms = 20 ms + alpha real = 1 + mu_plus real = 1 + mu_minus real = 1 + Wmax real = 100. + Wmin real = 0. equations: - pre_tr' = -pre_tr / tau_tr_pre - post_tr' = -post_tr / tau_tr_post - - internals: - tau_s 1/ms = (tau_c + tau_n) / (tau_c * tau_n) + pre_trace' = -pre_trace / tau_tr_pre + post_trace' = -post_trace / tau_tr_post input: pre_spikes <- spike post_spikes <- spike - mod_spikes <- spike output: spike(weight real, delay ms) - onReceive(mod_spikes): - n += A_vt / tau_n - onReceive(post_spikes): - post_tr += 1. + post_trace += 1 - # facilitation - c += A_plus * pre_tr + # potentiate synapse + w_ real = Wmax * ( w / Wmax + n * (lambda * ( 1. - ( w / Wmax ) )**mu_plus * pre_trace )) + w = min(Wmax, w_) onReceive(pre_spikes): - pre_tr += 1. + pre_trace += 1 - # depression - c -= A_minus * post_tr + # depress synapse + w_ real = Wmax * ( w / Wmax - n * ( alpha * lambda * ( w / Wmax )**mu_minus * post_trace )) + w = max(Wmin, w_) # deliver spike to postsynaptic partner emit_spike(w, d) - # update from time t to t + timestep() update: integrate_odes() - - # timestep() returns the timestep to be made (in units of time) - # the sequence here matters: the update step for w requires the "old" values of c and n - w -= c * ( n / tau_s * expm1( -tau_s * timestep() ) \ - - b * tau_c * expm1( -timestep() / tau_c )) - w = max(0., w) - c = c * exp(-timestep() / tau_c) - n = n * exp(-timestep() / tau_n) -""" \ No newline at end of file diff --git a/doc/tutorials/cart_pole_reinforcement_learning/polebalancing.ipynb b/doc/tutorials/cart_pole_reinforcement_learning/polebalancing.ipynb index 330653f73..0f7dbe11b 100644 --- a/doc/tutorials/cart_pole_reinforcement_learning/polebalancing.ipynb +++ b/doc/tutorials/cart_pole_reinforcement_learning/polebalancing.ipynb @@ -1645,16 +1645,19 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "99614c94", "metadata": {}, "outputs": [ { - "ename": "SyntaxError", - "evalue": "invalid syntax (1176944502.py, line 45)", + "ename": "NameError", + "evalue": "name 'Agent' is not defined", "output_type": "error", "traceback": [ - "\u001b[0;36m Cell \u001b[0;32mIn [2], line 45\u001b[0;36m\u001b[0m\n\u001b[0;31m self.input_population[box[0], box[1], box[2], box[3]].I_e = ??? # a current to make the neuron fire at a \"reasonable\" rate (like 10 Hz)\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [3], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnest\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSpikingAgent\u001b[39;00m(\u001b[43mAgent\u001b[49m):\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, initial_state: Tuple[\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(initial_state)\n", + "\u001b[0;31mNameError\u001b[0m: name 'Agent' is not defined" ] } ], @@ -1665,8 +1668,7 @@ "class SpikingAgent(Agent):\n", " def __init__(self, initial_state: Tuple[float,float,float,float]) -> None:\n", " super().__init__(initial_state)\n", - "\n", - " # ...\n", + " self.construct_neural_network()\n", " \n", " def get_state_neuron(self, state) -> int:\n", " idx = 0\n", @@ -1694,22 +1696,60 @@ " self.input_population[idx[0], idx[1], idx[2], idx[3]] = nest.Create(neuron_model_name)\n", " \"\"\"\n", " input_size = self.dimensions[0] * self.dimensions[1] * self.dimensions[2] * self.dimensions[3]\n", - " self.input_population = nest.Create(input_layer_neuron_model_name, input_size)\n", - " self.output_population = nest.Create(output_layer_neuron_model_name, 2) #2? 10?\n", + " \n", + " self.volume_transmitter = nest.Create(\"volume_transmitter\")\n", + " nest.CopyModel(output_layer_synapse_model_name, \"stdp_dopa_nestml\",\n", + " {\"volume_transmitter\": vt})\n", "\n", + " self.output_population_left = nest.Create(output_layer_neuron_model_name, 10) #2? 10?\n", + " self.output_population_right = nest.Create(output_layer_neuron_model_name, 10) #2? 10?\n", " \n", + " nest.Connect(self.input_population, self.output_population_left, syn_spec={\"synapse_model\": output_layer_synapse_model_name})\n", + " nest.Connect(self.input_population, self.output_population_right, syn_spec={\"synapse_model\": output_layer_synapse_model_name})\n", + "\n", + " self.output_population_spike_recorder_left = nest.Create(\"spike_recorder\")\n", + " nest.Connect(self.output_population_left, self.output_population_spike_recorder_left)\n", + "\n", + " self.output_population_spike_recorder_right = nest.Create(\"spike_recorder\")\n", + " nest.Connect(self.output_population_right, self.output_population_spike_recorder_right)\n", "\n", " def update(self, next_state: Tuple[float,float,float,float]):\n", " box = self.get_box(next_state) \n", " \n", - " # set input current on the neuron (I_e) to make it fire (at firing rate = ???)\n", - " self.input_population[box[0], box[1], box[2], box[3]].I_e = ??? # a current to make the neuron fire at a \"reasonable\" rate (like 10 Hz)\n", - "\n" + " # make the correct input neuron fire\n", + " self.input_population.firing_rate = 0.\n", + " self.input_population[box[0], box[1], box[2], box[3]].firing_rate = 10.\n", + "\n", + " nest.Simulate(nest.resolution)\n", + " \n", + " # measure output layer spikes\n", + " output_layer_spike_times_left = self.output_population_spike_recorder_left.get(\"events\")[\"times\"]\n", + " output_layer_spike_times_right = self.output_population_spike_recorder_right.get(\"events\")[\"times\"]\n", + " \n", + " # TODO measure the number of spikes in the last interval (interval = ???)\n", + " Q = ...\n", + " \n", + " # TODO take action based on measured output spikes\n", + " failure = a.update(...)\n", + "\n", + " # set new dopamine concentration on the synapses\n", + " if failure:\n", + " R = 0.\n", + " TD = -Q\n", + " else:\n", + " R = 1.\n", + " TD = gamma * Q + R - Q_prev\n", + "\n", + " syn = nest.GetConnections()\n", + " syn.n = TD\n", + " \n", + " \n", + " " ] }, { "cell_type": "markdown", - "id": "0490ced9", + "id": "566fa9c9", "metadata": {}, "source": [ "# Executing spiking version" @@ -1718,7 +1758,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c37f6cc0", + "id": "e4bda6d4", "metadata": {}, "outputs": [], "source": [