Added code for creating, training and evaluating MLPs.

MLP_plot_all.py

@@ -0,0 +1,132 @@
+import os
+from keras.models import model_from_json, Sequential
+from keras.layers import Activation, Dense, Dropout
+from keras.engine import Input, Model
+import keras.utils
+from keras.utils.visualize_util import plot
+import numpy as np
+import datetime
+import random
+from reader import Reader
+import matplotlib.pyplot as plt
+
+
+results_directory = '/home/orestis/net/MLPresults/'
+models_directory = '/home/orestis/net/MLPmodels/'
+
+
+def get_model(num_hid_layers, cells_per_layer, dropout_rate):
+	length = Reader.getInputShape()
+	model = Sequential()
+	model.add(Dense(cells_per_layer, input_shape=(length,), activation='relu'))
+	model.add(Dropout(dropout_rate))
+
+	for i in range(num_hid_layers):
+		model.add(Dense(cells_per_layer, activation='relu'))
+		model.add(Dropout(dropout_rate))
+
+	model.add(Dense(3, activation='softmax'))
+
+	model_name = models_directory + 'MLP.hidlay' + str(num_hid_layers) + '.cells' + str(cells_per_layer) + '.drop' + str(dropout_rate)
+	plot(model, to_file = model_name + '.png')
+
+	fp_model = open(model_name + '.json', 'w+')
+	fp_model.write(model.to_json())
+	fp_model.close()
+	return model
+
+
+def fit_and_eval(loss_function, optimizer, dropout_rate, dataset_name):
+	results_file = 	results_directory + dataset_name + '.' + loss_function + '.' + optimizer + '.Dropout' + str(dropout_rate)
+
+	write_results(results_file, 'HiddenLayers|CellsPerLayer|Accuracy')
+
+	# READ THE INPUT
+	fp_logfile = open('/home/orestis/net/working/logfile', 'a')
+	#reader = Reader(fp_logfile, False)
+	#(x_train, y_train), (x_test, y_test) = reader.getDataNormalized()
+
+	(x_train, y_train), (x_test, y_test) = Reader.getDataset(5)
+	x_train = x_train[0:1000,:]
+	y_train = y_train[0:1000]
+	x_test = x_test[0:1000,:]
+	y_test = y_test[0:1000]
+
+	num_classes = 3
+	nb_epoch = 15
+
+	lns = []
+	for num_hid_layers in [0,1,2,3,4]:
+		results = []
+		for cells_per_layer in [20,30,40,50,60]:
+			model = get_model(num_hid_layers, cells_per_layer, dropout_rate)
+			model.compile(optimizer=optimizer, loss=loss_function, metrics=['accuracy'])
+			model.fit(x_train, keras.utils.np_utils.to_categorical(y_train, num_classes), nb_epoch = nb_epoch, batch_size = 128, shuffle=True)
+			ev = model.evaluate(x = x_test, y = keras.utils.np_utils.to_categorical(y_test, num_classes), batch_size = 128)
+			results.append(ev[1])
+			#results.append(0.1*num_hid_layers + 0.001*cells_per_layer)		
+			write_results(results_file, str(num_hid_layers) + '|' + str(cells_per_layer) + '|' + str(ev[1]))
+		myplot = plt.subplot()
+		myplot.grid(True)
+		myplot.set_xlabel("Cells per Layer")
+		myplot.set_ylabel("Accuracy")
+		#myplot.set_xticklabels([20,30,40,50,60], rotation=45)
+		line = myplot.plot([20,30,40,50,60], results, label='hidLayers:'+str(num_hid_layers))		
+		lns = lns + line
+
+
+	loss_short = loss_function
+	if loss_short == 'categorical_crossentropy' or loss_short == 'binary_crossentropy':
+		loss_short = 'crossentropy'
+	plt.title('Dataset: ' + dataset_name + ' Model: MLP, Loss: ' + loss_short + ', Opt: ' + optimizer + ', Batch: 128, Dropout: ' + str(dropout_rate))
+	box = myplot.get_position()
+	myplot.set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8])	
+	labs = [l.get_label() for l in lns]
+	lgd = plt.legend(lns, labs,loc='upper center', bbox_to_anchor=(0.5, -0.15), fancybox=True, shadow=True, ncol=3) #
+	plt.savefig(results_directory + 'Dataset: ' + dataset_name + ' Model: MLP, Loss: ' + loss_short + ', Opt: ' + optimizer + ', Batch: 128, Dropout: ' + str(dropout_rate) + '.png')
+	plt.clf()
+
+
+def write_results(results_file, text):
+	fp_results = open(results_file, 'a')
+	fp_results.write(text +'\n')	
+	fp_results.close()
+
+
+
+
+#fit_and_eval('mse', 'sgd', 0)
+#fit_and_eval('mse', 'sgd', 0.2)
+#fit_and_eval('mse', 'sgd', 0.4)
+#fit_and_eval('mse', 'sgd', 0.6)
+#fit_and_eval('mse', 'sgd', 0.8)
+
+#fit_and_eval('mae', 'sgd', 0)
+#fit_and_eval('mae', 'sgd', 0.2)
+#fit_and_eval('mae', 'sgd', 0.4)
+#fit_and_eval('mae', 'sgd', 0.6)
+#fit_and_eval('mae', 'sgd', 0.8)
+
+#fit_and_eval('categorical_crossentropy', 'sgd', 0)
+#fit_and_eval('categorical_crossentropy', 'sgd', 0.2)
+#fit_and_eval('categorical_crossentropy', 'sgd', 0.4)
+#fit_and_eval('categorical_crossentropy', 'sgd', 0.6)
+#fit_and_eval('categorical_crossentropy', 'sgd', 0.8)
+
+#fit_and_eval('categorical_crossentropy', 'rmsprop', 0)
+#fit_and_eval('categorical_crossentropy', 'rmsprop', 0.2)
+#fit_and_eval('categorical_crossentropy', 'rmsprop', 0.4, '')
+#fit_and_eval('categorical_crossentropy', 'rmsprop', 0.6)
+#fit_and_eval('categorical_crossentropy', 'rmsprop', 0.8)
+
+#fit_and_eval('categorical_crossentropy', 'adagrad', 0)
+#fit_and_eval('categorical_crossentropy', 'adagrad', 0.2)
+#fit_and_eval('categorical_crossentropy', 'adagrad', 0.4, '')
+#fit_and_eval('categorical_crossentropy', 'adagrad', 0.6)
+#fit_and_eval('categorical_crossentropy', 'adagrad', 0.8)
+
+# Dataset 5 with MLP
+
+fit_and_eval('mse', 'sgd', 0.4, 'Dataset5')
+fit_and_eval('categorical_crossentropy', 'rmsprop', 0.4, 'Dataset5')
+

MLPmodels/MLP.hidlay0.cells20.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_101", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_76", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_102", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay0.cells30.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_103", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_77", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_104", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay0.cells40.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_105", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_78", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_106", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay0.cells50.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_107", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 50, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_79", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_108", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay0.cells60.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_109", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 60, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_80", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_110", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay1.cells20.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_111", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_81", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_112", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_82", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_113", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay1.cells30.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_114", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_83", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_115", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_84", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_116", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay1.cells40.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_117", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_85", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_118", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_86", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_119", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay1.cells50.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_120", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 50, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_87", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_121", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 50, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_88", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_122", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay1.cells60.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_123", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 60, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_89", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_124", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 60, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_90", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_125", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay2.cells20.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_126", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_91", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_127", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_92", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_128", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 20, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_93", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_129", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay2.cells30.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_130", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_94", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_131", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_95", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_132", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 30, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_96", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_133", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}

MLPmodels/MLP.hidlay2.cells40.drop0.4.json

@@ -0,0 +1 @@
+{"keras_version": "1.1.1", "class_name": "Sequential", "config": [{"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_134", "init": "glorot_uniform", "b_regularizer": null, "batch_input_shape": [null, 79], "input_dtype": "float32", "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "bias": true, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_97", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_135", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_98", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_136", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "relu", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 40, "W_regularizer": null}}, {"class_name": "Dropout", "config": {"name": "dropout_99", "p": 0.4, "trainable": true}}, {"class_name": "Dense", "config": {"W_constraint": null, "name": "dense_137", "init": "glorot_uniform", "b_regularizer": null, "bias": true, "b_constraint": null, "activation": "softmax", "input_dim": null, "trainable": true, "activity_regularizer": null, "output_dim": 3, "W_regularizer": null}}]}