"Added functions for testing different batch and epoch sizes"

models/autoencoder.py

@@ -8,6 +8,7 @@ from tensorflow.keras.models import Model
 import misc
 import defs
 
+import pyswarms as ps
 latent_dim = 64
 
 print("# GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
@@ -141,6 +142,155 @@ def view_encoder(encoder, N, samples=1000):
 
     pass
 
+def test_batch_sizes(autoencoder, sizes):
+    accuracy = []
+    for batch_size in sizes:
+        autoencoder.fit(x_train_ho, x_train_ho,
+                        epochs=1,
+                        batch_size=batch_size,
+                        shuffle=False,
+                        validation_data=(x_test_ho, x_test_ho))
+
+        encoded_data = autoencoder.encoder(x_test_ho)
+        decoded_data = autoencoder.decoder(encoded_data).numpy()
+
+        result = misc.int2bit_array(decoded_data.argmax(axis=1), n)
+        print("Accuracy: %.4f" % accuracy_score(x_test_array, result.reshape(-1, )))
+        accuracy.append(accuracy_score(x_test_array, result.reshape(-1, )))
+    plt.plot(sizes,accuracy)
+    plt.xscale('log')
+    plt.grid()
+    plt.xlabel('batch size')
+    plt.ylabel('Accuracy')
+    plt.legend()
+    plt.show()
+
+def test_epoch_sizes(autoencoder, sizes):
+    accuracy = []
+    for epoch_size in sizes:
+        autoencoder.fit(x_train_ho, x_train_ho,
+                        epochs=epoch_size,
+                        shuffle=False,
+                        validation_data=(x_test_ho, x_test_ho))
+
+        encoded_data = autoencoder.encoder(x_test_ho)
+        decoded_data = autoencoder.decoder(encoded_data).numpy()
+
+        result = misc.int2bit_array(decoded_data.argmax(axis=1), n)
+        print("Accuracy: %.4f" % accuracy_score(x_test_array, result.reshape(-1, )))
+        accuracy.append(accuracy_score(x_test_array, result.reshape(-1, )))
+    plt.plot(sizes,accuracy)
+    plt.xscale('log')
+    plt.grid()
+    plt.xlabel('epoch size')
+    plt.ylabel('Accuracy')
+    plt.legend()
+    plt.show()
+
+def sigmoid(z):
+    return 1. / (1. + np.exp(-z))
+
+# Forward propagation
+def logits_function(params, n, train):
+    """Forward propagation as objective function
+
+    This computes for the forward propagation of the neural network, as
+    well as the loss. It receives a set of parameters that must be
+    rolled-back into the corresponding weights and biases.
+
+    Inputs
+    ------
+    params: np.ndarray
+        The dimensions should include an unrolled version of the
+        weights and biases.
+
+    Returns
+    -------
+    float
+        The computed negative log-likelihood loss given the parameters
+    """
+    # Neural network architecture
+    n_inputs = 2 ** n
+    n_hidden = 2 ** (n + 1)
+    n_classes = 2
+
+    # Roll-back the weights and biases
+    W1 = params[0:n_inputs*n_hidden].reshape((n_inputs,n_hidden))
+    b1 = params[n_inputs*n_hidden:n_inputs*n_hidden+n_hidden].reshape((n_hidden,))
+    W2 = params[n_inputs*n_hidden+n_hidden:n_inputs*n_hidden+n_hidden+n_hidden*n_classes].reshape((n_hidden,n_classes))
+    b2 = params[n_inputs*n_hidden+n_hidden+n_hidden*n_classes:n_inputs*n_hidden+n_hidden+n_hidden*n_classes+n_classes].reshape((n_classes,))
+
+    # Perform forward propagation
+    z1 = train.dot(W1) + b1  # Pre-activation in Layer 1
+    a1 = np.tanh(z1)     # Activation in Layer 1
+    z2 = a1.dot(W2) + b2 # Pre-activation in Layer 2
+    logits = z2          # Logits for Layer 2
+
+    return logits
+
+# Forward propagation
+def forward_prop(params, n, train, samples):
+    """Forward propagation as objective function
+
+    This computes for the forward propagation of the neural network, as
+    well as the loss.
+
+    Inputs
+    ------
+    params: np.ndarray
+        The dimensions should include an unrolled version of the
+        weights and biases.
+
+    Returns
+    -------
+    float
+        The computed negative log-likelihood loss given the parameters
+    """
+
+    logits = logits_function(params, n, train)
+
+    # Compute for the softmax of the logits
+    exp_scores = np.exp(logits)
+    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
+
+    # Compute for the negative log likelihood
+
+    corect_logprobs = -np.log(probs[range(samples), train])
+    loss = np.sum(corect_logprobs) / samples
+
+    return loss
+
+def f(x, n, train, samples):
+    """Higher-level method to do forward_prop in the
+    whole swarm.
+
+    Inputs
+    ------
+    x: numpy.ndarray of shape (n_particles, dimensions)
+        The swarm that will perform the search
+
+    Returns
+    -------
+    numpy.ndarray of shape (n_particles, )
+        The computed loss for each particle
+    """
+    n_particles = x.shape[0]
+    j = [forward_prop(x[i], n, train, samples) for i in range(n_particles)]
+    return np.array(j)
+
+def predict(pos):
+    """
+    Use the trained weights to perform class predictions.
+
+    Inputs
+    ------
+    pos: numpy.ndarray
+        Position matrix found by the swarm. Will be rolled
+        into weights and biases.
+    """
+    logits = logits_function(pos)
+    y_pred = np.argmax(logits, axis=1)
+    return y_pred
 
 if __name__ == '__main__':
     # (x_train, _), (x_test, _) = fashion_mnist.load_data()
@@ -153,16 +303,46 @@ if __name__ == '__main__':
 
     n = 4
 
-    samples = 1e6
+    n_inputs = 2 ** n
+    n_hidden = 2 ** (n + 1)
+    n_classes = 2
+
+    samples = 1e5
+
+
     x_train = misc.generate_random_bit_array(samples).reshape((-1, n))
     x_train_ho = misc.bit_matrix2one_hot(x_train)
     x_test_array = misc.generate_random_bit_array(samples * 0.3)
     x_test = x_test_array.reshape((-1, n))
     x_test_ho = misc.bit_matrix2one_hot(x_test)
 
+    """
+    # Initialize swarm
+    options = {'c1': 0.5, 'c2': 0.3, 'w': 0.9}
+
+    # Call instance of PSO
+    dimensions = (n_inputs * n_hidden) + (n_hidden * n_classes) + n_hidden + n_classes
+    optimizer = ps.single.GlobalBestPSO(n_particles=100, dimensions=dimensions, options=options)
+
+    # Perform optimization
+    cost, pos = optimizer.optimize(f(x, n, x_train_ho, samples), iters=1000)
+
+    print((predict(pos) == x_train_ho).mean())
+
+    pass
+
+"""
     autoencoder = Autoencoder(n, -8)
     autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError())
 
+    batch_sizes = [10,20,30,40, 100,200,300,400,500, 1000,2000,3000,4000,5000]
+    epoch_sizes = [1, 10, 20, 50, 100, 200, 500]
+
+    test_epoch_sizes(autoencoder, epoch_sizes)
+
+    pass
+
+"""
     autoencoder.fit(x_train_ho, x_train_ho,
                     epochs=1,
                     shuffle=False,
@@ -174,5 +354,5 @@ if __name__ == '__main__':
     result = misc.int2bit_array(decoded_data.argmax(axis=1), n)
     print("Accuracy: %.4f" % accuracy_score(x_test_array, result.reshape(-1, )))
     view_encoder(autoencoder.encoder, n)
+"""
 
-    pass

models/swarm_test.py

@@ -11,16 +11,16 @@ import misc
 data = load_iris()
 
 # Store the features as X and the labels as y
-#X = data.data
-#y = data.target
+X = data.data
+y = data.target
 
 n = 4
-
+print(y.shape)
 n_inputs = 2 ** n
 n_hidden = 2 ** (n + 1)
-n_classes = 2
+n_classes = 2 ** n
 
-samples = 1000
+samples = 100000
 
 
 x_train = misc.generate_random_bit_array(samples).reshape((-1, n))
@@ -28,6 +28,10 @@ x_train_ho = misc.bit_matrix2one_hot(x_train)
 x_test_array = misc.generate_random_bit_array(samples * 0.3)
 x_test = x_test_array.reshape((-1, n))
 x_test_ho = misc.bit_matrix2one_hot(x_test)
+print(x_train.shape)
+
+print(x_train_ho.shape)
+
 
 def logits_function(p):
     """ Calculate roll-back the weights and biases
@@ -46,7 +50,7 @@ def logits_function(p):
     # Neural network architecture
     n_inputs = 2 ** n
     n_hidden = 2 ** (n + 1)
-    n_classes = 2
+    n_classes = 2 ** n
 
     # Roll-back the weights and biases
     W1 = p[0:n_inputs * n_hidden].reshape((n_inputs, n_hidden))
@@ -92,8 +96,8 @@ def forward_prop(params):
 
     # Compute for the negative log likelihood
 
-    corect_logprobs = -np.log(probs[range(samples), x_train_ho])
-    loss = np.sum(corect_logprobs) / samples
+    corect_logprobs = -np.log(probs[range(int(samples/n)), x_train_ho.shape[1]-1])
+    loss = np.sum(corect_logprobs) / samples/n
 
     return loss
 
@@ -123,7 +127,7 @@ dimensions = (n_inputs * n_hidden) + (n_hidden * n_classes) + n_hidden + n_class
 optimizer = ps.single.GlobalBestPSO(n_particles=100, dimensions=dimensions, options=options)
 
 # Perform optimization
-cost, pos = optimizer.optimize(f, iters=1000)
+cost, pos = optimizer.optimize(f, iters=2)
 
 def predict(pos):
     """
@@ -136,7 +140,10 @@ def predict(pos):
         into weights and biases.
     """
     logits = logits_function(pos)
+    print(logits.shape)
     y_pred = np.argmax(logits, axis=1)
+    print(y_pred)
     return y_pred
 
-print((predict(pos) == x_train_ho).mean())
+print((predict(pos) == x_train_ho))
+    #.mean()

models/tutorial.py

@@ -0,0 +1,124 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.datasets import load_iris
+
+
+# Import PySwarms
+import pyswarms as ps
+
+data = load_iris()
+
+
+# Store the features as X and the labels as y
+X = data.data
+y = data.target
+
+n_inputs = 4
+n_hidden = 20
+n_classes = 3
+
+num_samples = 150
+
+
+def logits_function(p):
+    """ Calculate roll-back the weights and biases
+
+    Inputs
+    ------
+    p: np.ndarray
+        The dimensions should include an unrolled version of the
+        weights and biases.
+
+    Returns
+    -------
+    numpy.ndarray of logits for layer 2
+
+    """
+    # Roll-back the weights and biases
+    W1 = p[0:80].reshape((n_inputs,n_hidden))
+    b1 = p[80:100].reshape((n_hidden,))
+    W2 = p[100:160].reshape((n_hidden,n_classes))
+    b2 = p[160:163].reshape((n_classes,))
+
+    # Perform forward propagation
+    z1 = X.dot(W1) + b1  # Pre-activation in Layer 1
+    a1 = np.tanh(z1)     # Activation in Layer 1
+    logits = a1.dot(W2) + b2 # Pre-activation in Layer 2
+    return logits          # Logits for Layer 2
+
+# Forward propagation
+def forward_prop(params):
+    """Forward propagation as objective function
+
+    This computes for the forward propagation of the neural network, as
+    well as the loss.
+
+    Inputs
+    ------
+    params: np.ndarray
+        The dimensions should include an unrolled version of the
+        weights and biases.
+
+    Returns
+    -------
+    float
+        The computed negative log-likelihood loss given the parameters
+    """
+
+    logits = logits_function(params)
+
+    # Compute for the softmax of the logits
+    exp_scores = np.exp(logits)
+    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
+
+    # Compute for the negative log likelihood
+
+    corect_logprobs = -np.log(probs[range(num_samples), y])
+    loss = np.sum(corect_logprobs) / num_samples
+
+    return loss
+
+
+def f(x):
+    """Higher-level method to do forward_prop in the
+    whole swarm.
+
+    Inputs
+    ------
+    x: numpy.ndarray of shape (n_particles, dimensions)
+        The swarm that will perform the search
+
+    Returns
+    -------
+    numpy.ndarray of shape (n_particles, )
+        The computed loss for each particle
+    """
+    n_particles = x.shape[0]
+    j = [forward_prop(x[i]) for i in range(n_particles)]
+    return np.array(j)
+
+# Initialize swarm
+options = {'c1': 0.5, 'c2': 0.3, 'w':0.9}
+
+# Call instance of PSO
+dimensions = (n_inputs * n_hidden) + (n_hidden * n_classes) + n_hidden + n_classes
+optimizer = ps.single.GlobalBestPSO(n_particles=100, dimensions=dimensions, options=options)
+
+# Perform optimization
+cost, pos = optimizer.optimize(f, iters=1000)
+
+def predict(pos):
+    """
+    Use the trained weights to perform class predictions.
+
+    Inputs
+    ------
+    pos: numpy.ndarray
+        Position matrix found by the swarm. Will be rolled
+        into weights and biases.
+    """
+    logits = logits_function(pos)
+    y_pred = np.argmax(logits, axis=1)
+    return y_pred
+
+(predict(pos) == y).mean()