Add all of my completed homeworks

hw10_tensorflow/carl - 04 - tensorflow/cnn_mnist.py

+import tflearn
+from tflearn.layers.core import input_data, dropout, fully_connected
+from tflearn.layers.conv import conv_2d, max_pool_2d
+from tflearn.layers.normalization import local_response_normalization
+from tflearn.layers.estimator import regression
+
+# Data loading and preprocessing
+import tflearn.datasets.mnist as mnist
+X, Y, testX, testY = mnist.load_data(one_hot=True)
+X = X.reshape([-1, 28, 28, 1])
+testX = testX.reshape([-1, 28, 28, 1])
+
+# Building convolutional network
+network = input_data(shape=[None, 28, 28, 1], name='input')
+network = conv_2d(network, 32, 3, activation='relu', regularizer='L2')
+network = max_pool_2d(network, 2)
+network = local_response_normalization(network)
+network = fully_connected(network, 128, activation='tanh')
+network = dropout(network, 0.8)
+network = fully_connected(network, 256, activation='tanh')
+network = dropout(network, 0.8)
+network = fully_connected(network, 10, activation='softmax')
+network = regression(network, optimizer='adam', learning_rate=0.01,
+                     loss='categorical_crossentropy', name='target')
+
+# Training
+model = tflearn.DNN(network, tensorboard_verbose=0)
+model.fit({'input': X}, {'target': Y}, n_epoch=20,
+          validation_set=({'input': testX}, {'target': testY}),
+          batch_size=100,
+          show_metric=True, run_id='cnn_mnist')

hw10_tensorflow/carl - 04 - tensorflow/run.sh

+#!/usr/bin/bash
+rm /tmp/tflearn_logs/cnn_mnist/*
+python cnn_mnist.py
+xdg-open http://localhost:6006
+tensorboard --logdir=/tmp/tflearn_logs/cnn_mnist

hw13_optimization/carl - 06 - optimization/autoencoder.jl

+using MLDatasets
+using Plots
+
+function flatten(X)
+  dims = size(X)
+  reshape(X, prod(dims[1:end-1]), dims[end])
+end
+
+function subset(X, y, labels::AbstractArray{Int64})
+  mask = in.(y, [labels])
+  X[:, mask], y[mask]
+end
+
+function subset(X, y, N::Integer)
+  mask = rand(1:length(y), N)
+  X[:, mask], y[mask]
+end
+
+function subset(X, y, labels)
+  mask = in.(y, [labels])
+  X[:, mask], y[mask]
+end
+
+type AutoEncoder{T<:Real}
+  N::Int64
+  W_enc::Matrix{T}
+  b_enc::Vector{T}
+  W_dec::Matrix{T}
+  b_dec::Vector{T}
+  AutoEncoder(N) = new(N)
+end
+
+relu(x) = max(0, x)
+drelu(x) = x .> 0
+logsig(n) = exp( -log(1 + exp(-n)) )
+dlogsig(n) = exp( n - 2 * log(exp(n) + 1) )
+# relu(x) = identity(x)
+# d_relu(x) = 1
+
+# logsig(n) = 1 / (1 + exp(-n))
+f_enc(n) = relu(n)
+df_enc(n) = drelu(n)
+f_dec(n) = logsig(n)
+df_dec(n) = dlogsig(n)
+
+
+AutoEncoder(N) = AutoEncoder{Float64}(N)
+
+encode(ae::AutoEncoder, X) = f_enc(ae.W_enc * X .+ ae.b_enc)
+decode(ae::AutoEncoder, Z) = f_dec(ae.W_dec * Z .+ ae.b_dec)
+
+function train!(ae::AutoEncoder, X; max_iterations=4000, learning_rate=0.1, batch_size=10, regularization=0.000, momentum=0.9)
+  D, T = size(X)
+
+  ae.W_enc = 0.001 * randn(ae.N, D)
+  ae.b_enc = 0.001 * randn(ae.N)
+  ae.W_dec = 0.001 * randn(D, ae.N)
+  ae.b_dec = 0.001 * randn(D)
+
+  f = (X) -> decode(ae, encode(ae, X))
+  l = (Y) -> 0.5 / T * sumabs2(f(Y) - X)
+
+  ΔW_enc = 0.0
+  Δb_enc = 0.0
+  ΔW_dec = 0.0
+  Δb_dec = 0.0
+
+  @show l(X)
+  for i in 1:max_iterations
+    println("iteration $i")
+    batch_idx = rand(1:T, batch_size)
+    # batch_idx = 1:T
+    X_batch = X[:,batch_idx]
+    Z_batch = encode(ae, X_batch)
+    ∇l = f(X_batch) - X_batch
+    Δ_dec = 1 / T * ∇l .* f_dec(ae.W_dec * Z_batch .+ ae.b_dec)
+    Δ_enc = ae.W_dec' * Δ_dec .* f_enc(ae.W_enc * X_batch .+ ae.b_enc)
+    ∂W_enc = Δ_enc * X_batch'
+    ∂b_enc = vec(sum(Δ_enc, 2)) # Δ_enc * ones(D)
+    ∂W_dec = Δ_dec * Z_batch'
+    ∂b_dec = vec(sum(Δ_dec, 2)) # Δ_dec * ones(ae.N)
+
+    ΔW_enc = momentum * ΔW_enc + learning_rate * ∂W_enc + regularization * ae.W_enc
+    Δb_enc = momentum * Δb_enc + learning_rate * ∂b_enc
+    ΔW_dec = momentum * ΔW_dec + learning_rate * ∂W_dec + regularization * ae.W_dec
+    Δb_dec = momentum * Δb_dec + learning_rate * ∂b_dec
+
+    ae.W_enc -= ΔW_enc
+    ae.b_enc -= Δb_enc
+    ae.W_dec -= ΔW_dec
+    ae.b_dec -= Δb_dec
+    @show l(X)
+  end
+
+end
+
+function test()
+  X, y = MNIST.traindata()
+  X = convert(Array{Float32}, X)
+  X = flatten(X)
+  # X, y = subset(X, y, [5, 2])
+  X, y = subset(X, y, 5000)
+  encoder = AutoEncoder{Float32}(100)
+  train!(encoder, X)
+
+  # feats = reshape(encoder.W_enc', 28, 28, 100)
+  # heatmap(flipdim(reshape(permutedims(reshape(feats, 28, 28, 10, 10), (2, 3, 1, 4)), 280, 280), 1))
+  encoder
+end
+
+encoder = test()

hw2_autoencoders/carl - 01 - autoencoders/ae_encoder_mnist_visualize.jl

+using MXNet
+using Plots
+plotlyjs()
+
+#--------------------------------------------------------------------------------
+# extract parameters and visualize trained model
+
+# helper function for wisualizing weights
+include("visualize_mnist.jl")
+
+# data provider
+include("data_provider_mnist.jl")
+train_provider, eval_provider = get_mnist_providers(data_name=:data, label_name=:output_label, subset=[0, 1, 7])
+
+# load pre-trained model
+prefix = "ae_encoder_mnist"
+epochs = 1000
+model = cd("$(dirname(@__FILE__))/output/models") do
+  mx.load_checkpoint(prefix, epochs, MXNet.mx.FeedForward)
+end
+
+# predict model
+eval_labels = vec(convert(Array{Int64}, eval_provider.label_arrays[1]))
+predictions = mx.predict(model, eval_provider)
+W = copy(model.arg_params[:enc_0_1_weight])
+
+# plot
+cd("$(dirname(@__FILE__))/output/plots") do
+  p1 = scatter(predictions[1,:], predictions[2,:], group=eval_labels)
+  savefig(p1, "$(prefix)_prediction_scatter.pdf")
+
+  p2 = mnist_visualize(W)
+  savefig(p2, "$(prefix)_weights.pdf")
+end

hw2_autoencoders/carl - 01 - autoencoders/ae_full_mnist_visualize.jl

+using MXNet
+using Plots
+plotlyjs()
+
+#--------------------------------------------------------------------------------
+# extract parameters and visualize trained model
+
+# helper function for wisualizing weights
+include("visualize_mnist.jl")
+
+# data provider
+include("data_provider_mnist.jl")
+train_provider, eval_provider = get_mnist_providers(data_name=:data, label_name=:output_label, subset=[0, 1, 7])
+
+# load pre-trained model
+prefix = "ae_full_mnist"
+epochs = 1000
+model = cd("$(dirname(@__FILE__))/output/models") do
+  mx.load_checkpoint(prefix, epochs, MXNet.mx.FeedForward)
+end
+
+# predict model
+eval_data = eval_provider.data_arrays[1]
+predictions = mx.predict(model, eval_provider)
+
+# plot
+cd("$(dirname(@__FILE__))/output/plots") do
+  p1 = mnist_visualize(predictions[:,1:100])
+  savefig(p1, "$(prefix)_predictions.pdf")
+
+  p2 = mnist_visualize(eval_data[:,1:100])
+  savefig(p2, "$(prefix)_original.pdf")
+end

hw2_autoencoders/carl - 01 - autoencoders/ae_mnist.jl

+using MXNet
+
+#--------------------------------------------------------------------------------
+# define and train autoencoder
+
+# autoencoder factory functions
+include("autoencoder.jl")
+
+# data provider
+include("data_provider_mnist.jl")
+batch_size = 100
+train_provider, eval_provider = get_mnist_autoencoder_providers(batch_size, data_name=:data, label_name=:output_label, subset=[0, 1, 7])
+
+# build the autoencoder
+layer_dims = [784, 64, 256, 64, 2]
+ae, encoder = autoencoder(layer_dims, data_name=:data, label_name=:output_label)
+
+# setup model
+model = mx.FeedForward(ae, context=mx.gpu.(0))
+
+# optimization algorithm
+optimizer = mx.ADAM()
+# optimizer = mx.SGD(lr=0.1, momentum=0.9, weight_decay=0.00001)
+
+# fit parameters
+mx.fit(model, optimizer, train_provider, n_epoch=1000, eval_metric=mx.MSE())
+
+# create the encoder partial model
+encoder_model = mx.FeedForward(encoder)
+encoder_model.arg_params = filter((k,v) -> !startswith(string(k), "dec"), model.arg_params)
+encoder_model.aux_params = model.aux_params
+
+ae_prefix = "ae_full_mnist"
+encoder_prefix = "ae_encoder_mnist"
+
+# save models
+cd("$(dirname(@__FILE__))/output/models") do
+  mx.save_checkpoint(model, ae_prefix, optimizer.state)
+  mx.save_checkpoint(encoder_model, encoder_prefix, optimizer.state)
+end
+
+# save visualizations of the network
+cd("$(dirname(@__FILE__))/output/plots") do
+  write("$ae_prefix.dot", mx.to_graphviz(ae))
+  run(`dot -Tpdf $ae_prefix.dot -o $ae_prefix.pdf`)
+  run(`rm $ae_prefix.dot`)
+
+  write("$encoder_prefix.dot", mx.to_graphviz(encoder))
+  run(`dot -Tpdf $encoder_prefix.dot -o $encoder_prefix.pdf`)
+  run(`rm $encoder_prefix.dot`)
+end

hw2_autoencoders/carl - 01 - autoencoders/autoencoder.jl

+# autoencoder factory functions
+
+# factory for single layer
+function build_layer(input, num_hidden, weight, name, act)
+  fc = mx.FullyConnected(input, weight, num_hidden=num_hidden, name="$(name)_fc")
+  if act != :none
+    fc = mx.Activation(fc, act_type=act, name="$(name)_act")
+  end
+  # fc = mx.BatchNorm(data=fc)
+  return fc
+end
+
+# recursive factory for autoencoder
+function build_autoencoder(input, layer_dims; inner_activation="relu", outer_activation="sigmoid", depth=0)
+  n = length(layer_dims)
+  if n < 2
+    return input, input
+  end
+  if n == 2
+    inner_activation = :none
+  end
+  name = "$(depth)_$(depth+1)"
+  W = mx.Variable("enc_$(name)_weight")
+  Wt = W'
+  mx.set_attr(Wt, :name, "enc_$(name)_weight")
+  encoder_input = build_layer(input, layer_dims[2], W, "enc_$(depth)_$(depth+1)", inner_activation)
+  decoder_input, encoder = build_autoencoder(encoder_input, layer_dims[2:end];
+                                             inner_activation=inner_activation,
+                                             outer_activation=inner_activation,
+                                             depth=depth+1)
+  decoder = build_layer(decoder_input, layer_dims[1], Wt, "dec_$(depth+1)_$(depth)", outer_activation)
+  return decoder, encoder
+end
+
+function autoencoder(layer_dims; data_name="data", label_name="output_label")
+  data = mx.Variable(data_name)
+  label = mx.Variable(label_name)
+  autoencoder, encoder = build_autoencoder(data, layer_dims)
+  cost = mx.LinearRegressionOutput(autoencoder, label, name="cost")
+  return cost, encoder
+end

hw2_autoencoders/carl - 01 - autoencoders/data_provider_mnist.jl

+using MNIST
+
+function get_mnist_autoencoder_providers(batch_size::Int; data_name=:data, label_name=:softmax_label, subset=0:9)
+  train_data, train_label = MNIST.traindata()
+  idx = BitArray(indexin(train_label, subset))
+  train_data = train_data[:,idx]
+  # train_label = train_data
+  scale!(train_data, 1/255)
+  train_provider = mx.ArrayDataProvider(data_name=>train_data, label_name=>train_data, batch_size=batch_size, shuffle=true)
+
+  eval_data, eval_label = MNIST.testdata()
+  idx = BitArray(indexin(eval_label, subset))
+  eval_data = eval_data[:,idx]
+  # eval_label = eval_data
+  scale!(eval_data, 1/255)
+  eval_provider = mx.ArrayDataProvider(data_name=>eval_data, label_name=>eval_data, batch_size=batch_size, shuffle=false)
+
+  return train_provider, eval_provider
+end
+
+function get_mnist_providers(batch_size::Int=0; data_name=:data, label_name=:softmax_label, subset=0:9)
+  train_data, train_label = MNIST.traindata()
+  idx = BitArray(indexin(train_label, subset))
+  train_data = train_data[:,idx]
+  train_label = train_label[idx]
+  scale!(train_data, 1/255)
+  train_provider = mx.ArrayDataProvider(data_name=>train_data, label_name=>train_label, batch_size=batch_size, shuffle=true)
+
+  eval_data, eval_label = MNIST.testdata()
+  idx = BitArray(indexin(eval_label, subset))
+  eval_data = eval_data[:,idx]
+  eval_label = eval_label[idx]
+  scale!(eval_data, 1/255)
+  eval_provider = mx.ArrayDataProvider(data_name=>eval_data, label_name=>eval_label, batch_size=batch_size, shuffle=false)
+
+  return train_provider, eval_provider
+end

hw2_autoencoders/carl - 01 - autoencoders/output/models/ae_encoder_mnist-symbol.json

+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "data", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_0_1_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_0_1_fc_bias", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_0_1_fc", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": [[0, 0, 0], [1, 0, 0], [2, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_0_1_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[3, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_1_2_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_1_2_fc_bias", 
+      "attr": {"num_hidden": "256"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_1_2_fc", 
+      "attr": {"num_hidden": "256"}, 
+      "inputs": [[4, 0, 0], [5, 0, 0], [6, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_1_2_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[7, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_2_3_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_2_3_fc_bias", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_2_3_fc", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": [[8, 0, 0], [9, 0, 0], [10, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_2_3_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[11, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_3_4_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_3_4_fc_bias", 
+      "attr": {"num_hidden": "2"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_3_4_fc", 
+      "attr": {"num_hidden": "2"}, 
+      "inputs": [[12, 0, 0], [13, 0, 0], [14, 0, 0]]
+    }
+  ], 
+  "arg_nodes": [0, 1, 2, 5, 6, 9, 10, 13, 14], 
+  "node_row_ptr": [
+    0, 
+    1, 
+    2, 
+    3, 
+    4, 
+    5, 
+    6, 
+    7, 
+    8, 
+    9, 
+    10, 
+    11, 
+    12, 
+    13, 
+    14, 
+    15, 
+    16
+  ], 
+  "heads": [[15, 0, 0]], 
+  "attrs": {"mxnet_version": ["int", 903]}
+}
\ No newline at end of file

hw2_autoencoders/carl - 01 - autoencoders/output/models/ae_full_mnist-symbol.json

+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "data", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_0_1_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_0_1_fc_bias", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_0_1_fc", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": [[0, 0, 0], [1, 0, 0], [2, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_0_1_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[3, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_1_2_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_1_2_fc_bias", 
+      "attr": {"num_hidden": "256"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_1_2_fc", 
+      "attr": {"num_hidden": "256"}, 
+      "inputs": [[4, 0, 0], [5, 0, 0], [6, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_1_2_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[7, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_2_3_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_2_3_fc_bias", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": []
+    }, 
+    {
+      "op": "FullyConnected", 
+      "name": "enc_2_3_fc", 
+      "attr": {"num_hidden": "64"}, 
+      "inputs": [[8, 0, 0], [9, 0, 0], [10, 0, 0]]
+    }, 
+    {
+      "op": "Activation", 
+      "name": "enc_2_3_act", 
+      "attr": {"act_type": "relu"}, 
+      "inputs": [[11, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "enc_3_4_weight", 
+      "inputs": []
+    }, 
+    {
+      "op": "null",