Skip to content

Commit

Permalink
generate NESTML model code
Browse files Browse the repository at this point in the history
  • Loading branch information
C.A.P. Linssen committed Jan 9, 2025
1 parent d11ade7 commit b5c6dd4
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 55 deletions.
71 changes: 29 additions & 42 deletions doc/tutorials/cart_pole_reinforcement_learning/neuromodulated_stdp_synapse.nestml
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,55 @@
"""
model neuromodulated_stdp_synapse:
stdp - Synapse model for spike-timing dependent plasticity
#########################################################

...

"""
model neuromodulated_stdp_synapse:
state:
w real = 1.
n real = 0. # Neuromodulator concentration
c real = 0. # Eligibility trace
pre_tr real = 0.
post_tr real = 0.
w real = 1 # Synaptic weight
pre_trace real = 0.
post_trace real = 0.

parameters:
d ms = 1 ms
tau_tr_pre ms = 20 ms # STDP time constant for weight changes caused by pre-before-post spike pairings.
tau_tr_post ms = 20 ms # STDP time constant for weight changes caused by post-before-pre spike pairings.
tau_c ms = 1000 ms # Time constant of eligibility trace
tau_n ms = 200 ms # Time constant of dopaminergic trace
b real = 0. # Dopaminergic baseline concentration
Wmax real = 200. # Maximal synaptic weight
Wmin real = 0. # Minimal synaptic weight
A_plus real = 1. # Multiplier applied to weight changes caused by pre-before-post spike pairings. If b (dopamine baseline concentration) is zero, then A_plus is simply the multiplier for facilitation (as in the stdp_synapse model). If b is not zero, then A_plus will be the multiplier for facilitation only if n - b is positive, where n is the instantenous dopamine concentration in the volume transmitter. If n - b is negative, A_plus will be the multiplier for depression.
A_minus real = 1.5 # Multiplier applied to weight changes caused by post-before-pre spike pairings. If b (dopamine baseline concentration) is zero, then A_minus is simply the multiplier for depression (as in the stdp_synapse model). If b is not zero, then A_minus will be the multiplier for depression only if n - b is positive, where n is the instantenous dopamine concentration in the volume transmitter. If n - b is negative, A_minus will be the multiplier for facilitation.
A_vt real = 1. # Multiplier applied to dopa spikes
n real = 0. # neuromodulator concentration between 0 and 1
d ms = 1 ms # Synaptic transmission delay
lambda real = .01
tau_tr_pre ms = 20 ms
tau_tr_post ms = 20 ms
alpha real = 1
mu_plus real = 1
mu_minus real = 1
Wmax real = 100.
Wmin real = 0.

equations:
pre_tr' = -pre_tr / tau_tr_pre
post_tr' = -post_tr / tau_tr_post

internals:
tau_s 1/ms = (tau_c + tau_n) / (tau_c * tau_n)
pre_trace' = -pre_trace / tau_tr_pre
post_trace' = -post_trace / tau_tr_post

input:
pre_spikes <- spike
post_spikes <- spike
mod_spikes <- spike

output:
spike(weight real, delay ms)

onReceive(mod_spikes):
n += A_vt / tau_n

onReceive(post_spikes):
post_tr += 1.
post_trace += 1

# facilitation
c += A_plus * pre_tr
# potentiate synapse
w_ real = Wmax * ( w / Wmax + n * (lambda * ( 1. - ( w / Wmax ) )**mu_plus * pre_trace ))
w = min(Wmax, w_)

onReceive(pre_spikes):
pre_tr += 1.
pre_trace += 1

# depression
c -= A_minus * post_tr
# depress synapse
w_ real = Wmax * ( w / Wmax - n * ( alpha * lambda * ( w / Wmax )**mu_minus * post_trace ))
w = max(Wmin, w_)

# deliver spike to postsynaptic partner
emit_spike(w, d)

# update from time t to t + timestep()
update:
integrate_odes()

# timestep() returns the timestep to be made (in units of time)
# the sequence here matters: the update step for w requires the "old" values of c and n
w -= c * ( n / tau_s * expm1( -tau_s * timestep() ) \
- b * tau_c * expm1( -timestep() / tau_c ))
w = max(0., w)
c = c * exp(-timestep() / tau_c)
n = n * exp(-timestep() / tau_n)
"""
66 changes: 53 additions & 13 deletions doc/tutorials/cart_pole_reinforcement_learning/polebalancing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1645,16 +1645,19 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"id": "99614c94",
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "invalid syntax (1176944502.py, line 45)",
"ename": "NameError",
"evalue": "name 'Agent' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;36m Cell \u001b[0;32mIn [2], line 45\u001b[0;36m\u001b[0m\n\u001b[0;31m self.input_population[box[0], box[1], box[2], box[3]].I_e = ??? # a current to make the neuron fire at a \"reasonable\" rate (like 10 Hz)\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn [3], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnest\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSpikingAgent\u001b[39;00m(\u001b[43mAgent\u001b[49m):\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, initial_state: Tuple[\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m,\u001b[38;5;28mfloat\u001b[39m]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(initial_state)\n",
"\u001b[0;31mNameError\u001b[0m: name 'Agent' is not defined"
]
}
],
Expand All @@ -1665,8 +1668,7 @@
"class SpikingAgent(Agent):\n",
" def __init__(self, initial_state: Tuple[float,float,float,float]) -> None:\n",
" super().__init__(initial_state)\n",
"\n",
" # ...\n",
" self.construct_neural_network()\n",
" \n",
" def get_state_neuron(self, state) -> int:\n",
" idx = 0\n",
Expand Down Expand Up @@ -1694,22 +1696,60 @@
" self.input_population[idx[0], idx[1], idx[2], idx[3]] = nest.Create(neuron_model_name)\n",
" \"\"\"\n",
" input_size = self.dimensions[0] * self.dimensions[1] * self.dimensions[2] * self.dimensions[3]\n",
" self.input_population = nest.Create(input_layer_neuron_model_name, input_size)\n",
" self.output_population = nest.Create(output_layer_neuron_model_name, 2) #2? 10?\n",
" \n",
" self.volume_transmitter = nest.Create(\"volume_transmitter\")\n",
" nest.CopyModel(output_layer_synapse_model_name, \"stdp_dopa_nestml\",\n",
" {\"volume_transmitter\": vt})\n",
"\n",
" self.output_population_left = nest.Create(output_layer_neuron_model_name, 10) #2? 10?\n",
" self.output_population_right = nest.Create(output_layer_neuron_model_name, 10) #2? 10?\n",
" \n",
" nest.Connect(self.input_population, self.output_population_left, syn_spec={\"synapse_model\": output_layer_synapse_model_name})\n",
" nest.Connect(self.input_population, self.output_population_right, syn_spec={\"synapse_model\": output_layer_synapse_model_name})\n",
"\n",
" self.output_population_spike_recorder_left = nest.Create(\"spike_recorder\")\n",
" nest.Connect(self.output_population_left, self.output_population_spike_recorder_left)\n",
"\n",
" self.output_population_spike_recorder_right = nest.Create(\"spike_recorder\")\n",
" nest.Connect(self.output_population_right, self.output_population_spike_recorder_right)\n",
"\n",
" def update(self, next_state: Tuple[float,float,float,float]):\n",
" box = self.get_box(next_state) \n",
" \n",
" # set input current on the neuron (I_e) to make it fire (at firing rate = ???)\n",
" self.input_population[box[0], box[1], box[2], box[3]].I_e = ??? # a current to make the neuron fire at a \"reasonable\" rate (like 10 Hz)\n",
"\n"
" # make the correct input neuron fire\n",
" self.input_population.firing_rate = 0.\n",
" self.input_population[box[0], box[1], box[2], box[3]].firing_rate = 10.\n",
"\n",
" nest.Simulate(nest.resolution)\n",
" \n",
" # measure output layer spikes\n",
" output_layer_spike_times_left = self.output_population_spike_recorder_left.get(\"events\")[\"times\"]\n",
" output_layer_spike_times_right = self.output_population_spike_recorder_right.get(\"events\")[\"times\"]\n",
" \n",
" # TODO measure the number of spikes in the last interval (interval = ???)\n",
" Q = ...\n",
" \n",
" # TODO take action based on measured output spikes\n",
" failure = a.update(...)\n",
"\n",
" # set new dopamine concentration on the synapses\n",
" if failure:\n",
" R = 0.\n",
" TD = -Q\n",
" else:\n",
" R = 1.\n",
" TD = gamma * Q + R - Q_prev\n",
"\n",
" syn = nest.GetConnections()\n",
" syn.n = TD\n",
" \n",
" \n",
" "
]
},
{
"cell_type": "markdown",
"id": "0490ced9",
"id": "566fa9c9",
"metadata": {},
"source": [
"# Executing spiking version"
Expand All @@ -1718,7 +1758,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "c37f6cc0",
"id": "e4bda6d4",
"metadata": {},
"outputs": [],
"source": [
Expand Down

0 comments on commit b5c6dd4

Please sign in to comment.