commit 368d1342288e28a58cfd940311569845c9f6f3e7 Author: dLedda Date: Tue Jun 23 22:53:47 2020 +0200 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f5e96db --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +venv \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..5c98b42 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,2 @@ +# Default ignored files +/workspace.xml \ No newline at end of file diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..d5a12c0 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +MNIST \ No newline at end of file diff --git a/.idea/MNIST.iml b/.idea/MNIST.iml new file mode 100644 index 0000000..85c7612 --- /dev/null +++ b/.idea/MNIST.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..0eefe32 --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..207156a --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e7f736d --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/__pycache__/import_data.cpython-36.pyc b/__pycache__/import_data.cpython-36.pyc new file mode 100644 index 0000000..29a59f2 Binary files /dev/null and b/__pycache__/import_data.cpython-36.pyc differ diff --git a/__pycache__/multiclass_perceptron.cpython-36.pyc b/__pycache__/multiclass_perceptron.cpython-36.pyc new file mode 100644 index 0000000..518a3e5 Binary files /dev/null and b/__pycache__/multiclass_perceptron.cpython-36.pyc differ diff --git a/import_data.py b/import_data.py new file mode 100644 index 0000000..6270980 --- /dev/null +++ b/import_data.py @@ -0,0 +1,77 @@ +from PIL import Image +from typing import Union, List, Generator, Callable + + +BOX_SHADING = " ░▒▓██" +IMAGE_ROW_SIZE = 28 +IMAGE_COL_SIZE = 28 +IMAGE_SIZE = IMAGE_ROW_SIZE*IMAGE_COL_SIZE + + +def show_picture(img_bytes: Union[bytes, List[int]]): + img = Image.new("RGB", (IMAGE_ROW_SIZE, IMAGE_COL_SIZE), "black") + pixels = img.load() + for i in range(IMAGE_ROW_SIZE): + for j in range(IMAGE_COL_SIZE): + pixel = img_bytes[IMAGE_ROW_SIZE*i + j] + pixels[j, i] = (pixel, pixel, pixel) + img.show() + + +def print_img_to_console(img: Union[bytes, List[int]]): + for row_start in range(0, IMAGE_SIZE, IMAGE_ROW_SIZE): + print("".join([BOX_SHADING[pixel // 51]*2 for pixel in img[row_start:row_start + IMAGE_ROW_SIZE]])) + print() + + +def read_labels(file_location: str): + with open(file_location, 'rb') as img_file: + img_data = img_file.read() + num_items = int.from_bytes(img_data[4:8], byteorder="big") + for i in range(8, num_items): + yield int.from_bytes(img_data[i:i + 1], byteorder="big") + + +def read_imgs(file_location: str, as_bytes=False): + with open(file_location, 'rb') as img_file: + img_data = img_file.read() + num_items = int.from_bytes(img_data[4:8], byteorder="big") + num_rows = int.from_bytes(img_data[8:12], byteorder="big") + num_cols = int.from_bytes(img_data[12:16], byteorder="big") + img_size = num_rows*num_cols + start_byte = 16 + if as_bytes: + for end_byte in range(start_byte + img_size, num_items*img_size, img_size): + yield img_data[start_byte:end_byte] + start_byte = end_byte + else: + for end_byte in range(start_byte + img_size, num_items*img_size, img_size): + yield [pixel for pixel in img_data[start_byte:end_byte]] + start_byte = end_byte + + +def read_img_lbl_pairs(imgs_file: str, lbls_file: str): + for img, label in zip(read_imgs(imgs_file), read_labels(lbls_file)): + yield img, label + + +def test_x_y(num: int = -1) -> Callable[[], Generator]: + if num == -1: + num = 9992 + + def generator(): + for i, (img, lbl) in zip(range(num), read_img_lbl_pairs("t10k-images.idx3-ubyte", "t10k-labels.idx1-ubyte")): + yield img, lbl + + return generator + + +def train_x_y(num: int = -1) -> Callable[[], Generator]: + if num == -1: + num = 60000 + + def generator(): + for i, (img, lbl) in zip(range(num), read_img_lbl_pairs("train-images.idx3-ubyte", "train-labels.idx1-ubyte")): + yield img, lbl + + return generator diff --git a/main.py b/main.py new file mode 100644 index 0000000..7ed1437 --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +import torch +from multiclass_perceptron import train_and_test_multiclass_perceptron +from import_data import show_picture, test_x_y + +train_and_test_multiclass_perceptron() \ No newline at end of file diff --git a/mlp_network.py b/mlp_network.py new file mode 100644 index 0000000..a6d9ab9 --- /dev/null +++ b/mlp_network.py @@ -0,0 +1,118 @@ +import numpy as np +from recordclass import recordclass +from typing import NamedTuple, Tuple, List, Callable, Generator +from import_data import train_x_y, test_x_y +import sys + + +class LossFun(NamedTuple): + exec: Callable[[np.array, np.array], float] + deriv: Callable[[np.array, np.array], np.array] + + +def sum_squares_loss_func(predicted: np.array, gold: np.array) -> float: + return sum((predicted - gold) ** 2) + + +def sum_squares_loss_derivative(predicted: np.array, gold: np.array) -> np.array: + return 2 * (predicted - gold) + + +sum_squares_loss = LossFun(sum_squares_loss_func, sum_squares_loss_derivative) + + +def sigmoid(x: np.array) -> np.array: + return np.where(x >= 0, + 1 / (1 + np.exp(-x)), + np.exp(x) / (1 + np.exp(x))) + + +def sigmoid_deriv(x: np.array) -> np.array: + return sigmoid(x) * (1 - sigmoid(x)) + + +def softmax(x: np.array) -> np.array: + array_sum = sum(x) + return np.exp(x) / np.exp(array_sum) + + +class FFNeuralNetwork: + Layer = recordclass("Layer", "weights last_out last_linear") + + def __init__(self, structure: List[int], loss_fun: LossFun, learn_rate: float = 0.001): + self.learn_rate = learn_rate + self.loss_fun: LossFun = loss_fun + self.layers: List[FFNeuralNetwork.Layer] = [] + for i, layer_size in enumerate(structure[1:]): + self.layers.append( + FFNeuralNetwork.Layer( + weights=np.zeros([structure[i], layer_size]), + last_out=np.zeros(layer_size), + last_linear=np.zeros(layer_size))) + + def feed_forward(self, datum: List[float]): + out = datum + for i, layer in enumerate(self.layers): + layer.last_out = sigmoid(self.linear_forward(i, out)) + out = layer.last_out + return sigmoid(out) + + def linear_forward(self, layer_index, last_output: np.array): + layer = self.layers[layer_index] + result = np.dot(layer.weights.T, last_output) # + layer.bias + layer.last_linear = result + return result + + def predict(self, datum: List[float]): + return np.argmax(self.feed_forward(datum)) + + def calculate_loss(self, input_data: List[float], golden: List[int]): + return self.loss_fun.exec(self.feed_forward(input_data), np.array(golden)) + + def back_prop(self, input_data: List[int], golden: int, output: np.array): + golden = [1 if i == golden else 0 for i in range(len(output))] + + layer = self.layers[-1] + dloss_dout = self.loss_fun.deriv(output, golden) + dout_dlast_layer = sigmoid_deriv(layer.last_linear) + dlast_layer_dweights = self.layers[-2].last_out + dloss_dweights = np.outer(dlast_layer_dweights, (dloss_dout * dout_dlast_layer)) + layer.weights += self.learn_rate * dloss_dweights + + dloss_dprev = self.backprop_middle_layers(dloss_dweights) if len(self.layers) > 2 else dloss_dweights + + layer = self.layers[0] + dout_dlast_linear = sigmoid_deriv(layer.last_linear) + dlast_linear_dinput = input_data + dloss_dinput = np.outer(dlast_linear_dinput, np.dot(dout_dlast_linear, dloss_dprev)) + layer.weights += self.learn_rate * dloss_dinput + + def backprop_middle_layers(self, dloss_dprev: np.array): + for i, layer in enumerate(reversed(self.layers[1:-1])): + dout_dlast_layer = sigmoid_deriv(layer.last_linear) + dlast_layer_dweights = self.layers[i - 1].last_out + dloss_dweights = np.dot(dlast_layer_dweights, np.dot(dout_dlast_layer, dloss_dprev)) + layer.weights += self.learn_rate * dloss_dweights + dloss_dprev = dloss_dweights + return dloss_dprev + + def train(self, input_data: Callable[[], Generator], epochs: int = 4): + for epoch in range(epochs): + print(f"Training epoch: {epoch + 1}") + for datum, label in input_data(): + self.back_prop(datum, label, self.feed_forward(datum)) + + +def train_and_test_neural_network(): + model = FFNeuralNetwork([28**2, 100, 10], sum_squares_loss, 0.0001) + training_data_gen = train_x_y(1000) + test_data = test_x_y(10)() + model.train(training_data_gen, 5) + for test_datum, label in test_data: + print(model.feed_forward(test_datum), label) + np.set_printoptions(threshold=sys.maxsize) + print(model.layers[0].weights) + + +if __name__ == "__main__": + train_and_test_neural_network() diff --git a/multiclass_perceptron.py b/multiclass_perceptron.py new file mode 100644 index 0000000..15f8200 --- /dev/null +++ b/multiclass_perceptron.py @@ -0,0 +1,71 @@ +from typing import Tuple, List, Callable, Generator +from import_data import test_x_y, train_x_y, IMAGE_SIZE, print_img_to_console, show_picture +import numpy as np + + +class MulticlassPerceptron: + class Perceptron: + def __init__(self, input_size, learn_rate: float = 0.0001): + self.weights = np.random.rand(input_size + 1) * 2 - 1 + self.learn_rate = learn_rate + + def train(self, data_lbl_pairs: Callable[[], Generator], positive_label: int, iterations: int = 10): + for num_iter in range(iterations): + print(f"Iteration number: {num_iter + 1}") + for i, (x, y) in enumerate(data_lbl_pairs()): + prediction = self.output(x) + label = 1 if y == positive_label else -1 + correct = prediction >= 0 and label >= 0 or prediction < 0 and label < 0 + if not correct: + self.weights = self.weights + self.learn_rate * label * np.array([1.0] + x) + + def output(self, datum): + return np.dot(self.weights, np.array([1.0] + datum)) + + def get_normalised_weight_array(self): + return [int(x) for x in (((self.weights / np.max(self.weights) + 1) / 2) * 255)] + + def __init__(self, input_size: int, num_classes: int, learn_rate: float = 0.001): + self.classifiers = [self.Perceptron(input_size, learn_rate) for _ in range(num_classes)] + + def train(self, data_lbl_pairs: Callable[[], Generator], iterations: int = 10): + for i, classifier in enumerate(self.classifiers): + print(f"Training classifier for class {i}...") + classifier.train(data_lbl_pairs, i, iterations) + print() + + def output(self, datum: List[int]): + return list(map(lambda x: x.output(datum), self.classifiers)) + + def prediction(self, datum): + return max(list(range(10)), key=lambda x: self.output(datum)[x]) + + def view_for_classifier(self, classifier_index: int): + return self.classifiers[classifier_index].get_normalised_weight_array() + + +def train_and_test_multiclass_perceptron(iterations: int = 5, training_inputs: int = 5000, test_inputs: int = 1000): + print("Loading data") + training_data_gen = train_x_y(training_inputs) + print("Begin training model!") + model = MulticlassPerceptron(IMAGE_SIZE, 10) + model.train(training_data_gen, iterations) + print("Model successfully trained.") + print("Testing model...") + test_data = list(test_x_y(test_inputs)()) + n_correct = sum(model.prediction(x) == y for x, y in test_data) + accuracy = n_correct / len(test_data) + print(f"Accuracy: {accuracy} ({n_correct} correctly classified out of {len(test_data)} total test inputs.)") + for i in range(10): + print_img_to_console(model.view_for_classifier(i)) + + +def get_trained_digit_model(iterations: int = 5, training_inputs: int = 5000, test_inputs: int = 1000): + training_data_gen = train_x_y(training_inputs) + model = MulticlassPerceptron(IMAGE_SIZE, 10) + model.train(training_data_gen, iterations) + return model + + +if __name__ == "__main__": + train_and_test_multiclass_perceptron() diff --git a/t10k-images.idx3-ubyte b/t10k-images.idx3-ubyte new file mode 100644 index 0000000..1170b2c Binary files /dev/null and b/t10k-images.idx3-ubyte differ diff --git a/t10k-labels.idx1-ubyte b/t10k-labels.idx1-ubyte new file mode 100644 index 0000000..d1c3a97 Binary files /dev/null and b/t10k-labels.idx1-ubyte differ diff --git a/train-images.idx3-ubyte b/train-images.idx3-ubyte new file mode 100644 index 0000000..bbce276 Binary files /dev/null and b/train-images.idx3-ubyte differ diff --git a/train-labels.idx1-ubyte b/train-labels.idx1-ubyte new file mode 100644 index 0000000..d6b4c5d Binary files /dev/null and b/train-labels.idx1-ubyte differ