First commit

.gitignore

@@ -0,0 +1 @@
+venv

.idea/.gitignore

@@ -0,0 +1,2 @@
+# Default ignored files
+/workspace.xml

.idea/.name

@@ -0,0 +1 @@
+MNIST

.idea/MNIST.iml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,5 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="PROJECT_PROFILE" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (Perceptron)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/MNIST.iml" filepath="$PROJECT_DIR$/.idea/MNIST.iml" />
+    </modules>
+  </component>
+</project>

import_data.py

@@ -0,0 +1,77 @@
+from PIL import Image
+from typing import Union, List, Generator, Callable
+
+
+BOX_SHADING = " ░▒▓██"
+IMAGE_ROW_SIZE = 28
+IMAGE_COL_SIZE = 28
+IMAGE_SIZE = IMAGE_ROW_SIZE*IMAGE_COL_SIZE
+
+
+def show_picture(img_bytes: Union[bytes, List[int]]):
+    img = Image.new("RGB", (IMAGE_ROW_SIZE, IMAGE_COL_SIZE), "black")
+    pixels = img.load()
+    for i in range(IMAGE_ROW_SIZE):
+        for j in range(IMAGE_COL_SIZE):
+            pixel = img_bytes[IMAGE_ROW_SIZE*i + j]
+            pixels[j, i] = (pixel, pixel, pixel)
+    img.show()
+
+
+def print_img_to_console(img: Union[bytes, List[int]]):
+    for row_start in range(0, IMAGE_SIZE, IMAGE_ROW_SIZE):
+        print("".join([BOX_SHADING[pixel // 51]*2 for pixel in img[row_start:row_start + IMAGE_ROW_SIZE]]))
+    print()
+
+
+def read_labels(file_location: str):
+    with open(file_location, 'rb') as img_file:
+        img_data = img_file.read()
+        num_items = int.from_bytes(img_data[4:8], byteorder="big")
+        for i in range(8, num_items):
+            yield int.from_bytes(img_data[i:i + 1], byteorder="big")
+
+
+def read_imgs(file_location: str, as_bytes=False):
+    with open(file_location, 'rb') as img_file:
+        img_data = img_file.read()
+        num_items = int.from_bytes(img_data[4:8], byteorder="big")
+        num_rows = int.from_bytes(img_data[8:12], byteorder="big")
+        num_cols = int.from_bytes(img_data[12:16], byteorder="big")
+        img_size = num_rows*num_cols
+        start_byte = 16
+        if as_bytes:
+            for end_byte in range(start_byte + img_size, num_items*img_size, img_size):
+                yield img_data[start_byte:end_byte]
+                start_byte = end_byte
+        else:
+            for end_byte in range(start_byte + img_size, num_items*img_size, img_size):
+                yield [pixel for pixel in img_data[start_byte:end_byte]]
+                start_byte = end_byte
+
+
+def read_img_lbl_pairs(imgs_file: str, lbls_file: str):
+    for img, label in zip(read_imgs(imgs_file), read_labels(lbls_file)):
+        yield img, label
+
+
+def test_x_y(num: int = -1) -> Callable[[], Generator]:
+    if num == -1:
+        num = 9992
+
+    def generator():
+        for i, (img, lbl) in zip(range(num), read_img_lbl_pairs("t10k-images.idx3-ubyte", "t10k-labels.idx1-ubyte")):
+            yield img, lbl
+
+    return generator
+
+
+def train_x_y(num: int = -1) -> Callable[[], Generator]:
+    if num == -1:
+        num = 60000
+
+    def generator():
+        for i, (img, lbl) in zip(range(num), read_img_lbl_pairs("train-images.idx3-ubyte", "train-labels.idx1-ubyte")):
+            yield img, lbl
+
+    return generator

main.py

@@ -0,0 +1,5 @@
+import torch
+from multiclass_perceptron import train_and_test_multiclass_perceptron
+from import_data import show_picture, test_x_y
+
+train_and_test_multiclass_perceptron()

mlp_network.py

@@ -0,0 +1,118 @@
+import numpy as np
+from recordclass import recordclass
+from typing import NamedTuple, Tuple, List, Callable, Generator
+from import_data import train_x_y, test_x_y
+import sys
+
+
+class LossFun(NamedTuple):
+    exec: Callable[[np.array, np.array], float]
+    deriv: Callable[[np.array, np.array], np.array]
+
+
+def sum_squares_loss_func(predicted: np.array, gold: np.array) -> float:
+    return sum((predicted - gold) ** 2)
+
+
+def sum_squares_loss_derivative(predicted: np.array, gold: np.array) -> np.array:
+    return 2 * (predicted - gold)
+
+
+sum_squares_loss = LossFun(sum_squares_loss_func, sum_squares_loss_derivative)
+
+
+def sigmoid(x: np.array) -> np.array:
+    return np.where(x >= 0,
+                    1 / (1 + np.exp(-x)),
+                    np.exp(x) / (1 + np.exp(x)))
+
+
+def sigmoid_deriv(x: np.array) -> np.array:
+    return sigmoid(x) * (1 - sigmoid(x))
+
+
+def softmax(x: np.array) -> np.array:
+    array_sum = sum(x)
+    return np.exp(x) / np.exp(array_sum)
+
+
+class FFNeuralNetwork:
+    Layer = recordclass("Layer", "weights last_out last_linear")
+
+    def __init__(self, structure: List[int], loss_fun: LossFun, learn_rate: float = 0.001):
+        self.learn_rate = learn_rate
+        self.loss_fun: LossFun = loss_fun
+        self.layers: List[FFNeuralNetwork.Layer] = []
+        for i, layer_size in enumerate(structure[1:]):
+            self.layers.append(
+                FFNeuralNetwork.Layer(
+                    weights=np.zeros([structure[i], layer_size]),
+                    last_out=np.zeros(layer_size),
+                    last_linear=np.zeros(layer_size)))
+
+    def feed_forward(self, datum: List[float]):
+        out = datum
+        for i, layer in enumerate(self.layers):
+            layer.last_out = sigmoid(self.linear_forward(i, out))
+            out = layer.last_out
+        return sigmoid(out)
+
+    def linear_forward(self, layer_index, last_output: np.array):
+        layer = self.layers[layer_index]
+        result = np.dot(layer.weights.T, last_output)  # + layer.bias
+        layer.last_linear = result
+        return result
+
+    def predict(self, datum: List[float]):
+        return np.argmax(self.feed_forward(datum))
+
+    def calculate_loss(self, input_data: List[float], golden: List[int]):
+        return self.loss_fun.exec(self.feed_forward(input_data), np.array(golden))
+
+    def back_prop(self, input_data: List[int], golden: int, output: np.array):
+        golden = [1 if i == golden else 0 for i in range(len(output))]
+
+        layer = self.layers[-1]
+        dloss_dout = self.loss_fun.deriv(output, golden)
+        dout_dlast_layer = sigmoid_deriv(layer.last_linear)
+        dlast_layer_dweights = self.layers[-2].last_out
+        dloss_dweights = np.outer(dlast_layer_dweights, (dloss_dout * dout_dlast_layer))
+        layer.weights += self.learn_rate * dloss_dweights
+
+        dloss_dprev = self.backprop_middle_layers(dloss_dweights) if len(self.layers) > 2 else dloss_dweights
+
+        layer = self.layers[0]
+        dout_dlast_linear = sigmoid_deriv(layer.last_linear)
+        dlast_linear_dinput = input_data
+        dloss_dinput = np.outer(dlast_linear_dinput, np.dot(dout_dlast_linear, dloss_dprev))
+        layer.weights += self.learn_rate * dloss_dinput
+
+    def backprop_middle_layers(self, dloss_dprev: np.array):
+        for i, layer in enumerate(reversed(self.layers[1:-1])):
+            dout_dlast_layer = sigmoid_deriv(layer.last_linear)
+            dlast_layer_dweights = self.layers[i - 1].last_out
+            dloss_dweights = np.dot(dlast_layer_dweights, np.dot(dout_dlast_layer, dloss_dprev))
+            layer.weights += self.learn_rate * dloss_dweights
+            dloss_dprev = dloss_dweights
+        return dloss_dprev
+
+    def train(self, input_data: Callable[[], Generator], epochs: int = 4):
+        for epoch in range(epochs):
+            print(f"Training epoch: {epoch + 1}")
+            for datum, label in input_data():
+                self.back_prop(datum, label, self.feed_forward(datum))
+
+
+def train_and_test_neural_network():
+    model = FFNeuralNetwork([28**2, 100, 10], sum_squares_loss, 0.0001)
+    training_data_gen = train_x_y(1000)
+    test_data = test_x_y(10)()
+    model.train(training_data_gen, 5)
+    for test_datum, label in test_data:
+        print(model.feed_forward(test_datum), label)
+    np.set_printoptions(threshold=sys.maxsize)
+    print(model.layers[0].weights)
+
+
+if __name__ == "__main__":
+    train_and_test_neural_network()

multiclass_perceptron.py

@@ -0,0 +1,71 @@
+from typing import Tuple, List, Callable, Generator
+from import_data import test_x_y, train_x_y, IMAGE_SIZE, print_img_to_console, show_picture
+import numpy as np
+
+
+class MulticlassPerceptron:
+    class Perceptron:
+        def __init__(self, input_size, learn_rate: float = 0.0001):
+            self.weights = np.random.rand(input_size + 1) * 2 - 1
+            self.learn_rate = learn_rate
+
+        def train(self, data_lbl_pairs: Callable[[], Generator], positive_label: int, iterations: int = 10):
+            for num_iter in range(iterations):
+                print(f"Iteration number: {num_iter + 1}")
+                for i, (x, y) in enumerate(data_lbl_pairs()):
+                    prediction = self.output(x)
+                    label = 1 if y == positive_label else -1
+                    correct = prediction >= 0 and label >= 0 or prediction < 0 and label < 0
+                    if not correct:
+                        self.weights = self.weights + self.learn_rate * label * np.array([1.0] + x)
+
+        def output(self, datum):
+            return np.dot(self.weights, np.array([1.0] + datum))
+
+        def get_normalised_weight_array(self):
+            return [int(x) for x in (((self.weights / np.max(self.weights) + 1) / 2) * 255)]
+
+    def __init__(self, input_size: int, num_classes: int, learn_rate: float = 0.001):
+        self.classifiers = [self.Perceptron(input_size, learn_rate) for _ in range(num_classes)]
+
+    def train(self, data_lbl_pairs: Callable[[], Generator], iterations: int = 10):
+        for i, classifier in enumerate(self.classifiers):
+            print(f"Training classifier for class {i}...")
+            classifier.train(data_lbl_pairs, i, iterations)
+            print()
+
+    def output(self, datum: List[int]):
+        return list(map(lambda x: x.output(datum), self.classifiers))
+
+    def prediction(self, datum):
+        return max(list(range(10)), key=lambda x: self.output(datum)[x])
+
+    def view_for_classifier(self, classifier_index: int):
+        return self.classifiers[classifier_index].get_normalised_weight_array()
+
+
+def train_and_test_multiclass_perceptron(iterations: int = 5, training_inputs: int = 5000, test_inputs: int = 1000):
+    print("Loading data")
+    training_data_gen = train_x_y(training_inputs)
+    print("Begin training model!")
+    model = MulticlassPerceptron(IMAGE_SIZE, 10)
+    model.train(training_data_gen, iterations)
+    print("Model successfully trained.")
+    print("Testing model...")
+    test_data = list(test_x_y(test_inputs)())
+    n_correct = sum(model.prediction(x) == y for x, y in test_data)
+    accuracy = n_correct / len(test_data)
+    print(f"Accuracy: {accuracy} ({n_correct} correctly classified out of {len(test_data)} total test inputs.)")
+    for i in range(10):
+        print_img_to_console(model.view_for_classifier(i))
+
+
+def get_trained_digit_model(iterations: int = 5, training_inputs: int = 5000, test_inputs: int = 1000):
+    training_data_gen = train_x_y(training_inputs)
+    model = MulticlassPerceptron(IMAGE_SIZE, 10)
+    model.train(training_data_gen, iterations)
+    return model
+
+
+if __name__ == "__main__":
+    train_and_test_multiclass_perceptron()