Commit 56434052 authored by Carl Åkerlindh's avatar Carl Åkerlindh
Browse files

Add all of my completed homeworks

parent 5f31cfa2
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
# Data loading and preprocessing
import tflearn.datasets.mnist as mnist
X, Y, testX, testY = mnist.load_data(one_hot=True)
X = X.reshape([-1, 28, 28, 1])
testX = testX.reshape([-1, 28, 28, 1])
# Building convolutional network
network = input_data(shape=[None, 28, 28, 1], name='input')
network = conv_2d(network, 32, 3, activation='relu', regularizer='L2')
network = max_pool_2d(network, 2)
network = local_response_normalization(network)
network = fully_connected(network, 128, activation='tanh')
network = dropout(network, 0.8)
network = fully_connected(network, 256, activation='tanh')
network = dropout(network, 0.8)
network = fully_connected(network, 10, activation='softmax')
network = regression(network, optimizer='adam', learning_rate=0.01,
loss='categorical_crossentropy', name='target')
# Training
model = tflearn.DNN(network, tensorboard_verbose=0)
model.fit({'input': X}, {'target': Y}, n_epoch=20,
validation_set=({'input': testX}, {'target': testY}),
batch_size=100,
show_metric=True, run_id='cnn_mnist')
#!/usr/bin/bash
rm /tmp/tflearn_logs/cnn_mnist/*
python cnn_mnist.py
xdg-open http://localhost:6006
tensorboard --logdir=/tmp/tflearn_logs/cnn_mnist
using MLDatasets
using Plots
function flatten(X)
dims = size(X)
reshape(X, prod(dims[1:end-1]), dims[end])
end
function subset(X, y, labels::AbstractArray{Int64})
mask = in.(y, [labels])
X[:, mask], y[mask]
end
function subset(X, y, N::Integer)
mask = rand(1:length(y), N)
X[:, mask], y[mask]
end
function subset(X, y, labels)
mask = in.(y, [labels])
X[:, mask], y[mask]
end
type AutoEncoder{T<:Real}
N::Int64
W_enc::Matrix{T}
b_enc::Vector{T}
W_dec::Matrix{T}
b_dec::Vector{T}
AutoEncoder(N) = new(N)
end
relu(x) = max(0, x)
drelu(x) = x .> 0
logsig(n) = exp( -log(1 + exp(-n)) )
dlogsig(n) = exp( n - 2 * log(exp(n) + 1) )
# relu(x) = identity(x)
# d_relu(x) = 1
# logsig(n) = 1 / (1 + exp(-n))
f_enc(n) = relu(n)
df_enc(n) = drelu(n)
f_dec(n) = logsig(n)
df_dec(n) = dlogsig(n)
AutoEncoder(N) = AutoEncoder{Float64}(N)
encode(ae::AutoEncoder, X) = f_enc(ae.W_enc * X .+ ae.b_enc)
decode(ae::AutoEncoder, Z) = f_dec(ae.W_dec * Z .+ ae.b_dec)
function train!(ae::AutoEncoder, X; max_iterations=4000, learning_rate=0.1, batch_size=10, regularization=0.000, momentum=0.9)
D, T = size(X)
ae.W_enc = 0.001 * randn(ae.N, D)
ae.b_enc = 0.001 * randn(ae.N)
ae.W_dec = 0.001 * randn(D, ae.N)
ae.b_dec = 0.001 * randn(D)
f = (X) -> decode(ae, encode(ae, X))
l = (Y) -> 0.5 / T * sumabs2(f(Y) - X)
ΔW_enc = 0.0
Δb_enc = 0.0
ΔW_dec = 0.0
Δb_dec = 0.0
@show l(X)
for i in 1:max_iterations
println("iteration $i")
batch_idx = rand(1:T, batch_size)
# batch_idx = 1:T
X_batch = X[:,batch_idx]
Z_batch = encode(ae, X_batch)
∇l = f(X_batch) - X_batch
Δ_dec = 1 / T * ∇l .* f_dec(ae.W_dec * Z_batch .+ ae.b_dec)
Δ_enc = ae.W_dec' * Δ_dec .* f_enc(ae.W_enc * X_batch .+ ae.b_enc)
∂W_enc = Δ_enc * X_batch'
∂b_enc = vec(sum(Δ_enc, 2)) # Δ_enc * ones(D)
∂W_dec = Δ_dec * Z_batch'
∂b_dec = vec(sum(Δ_dec, 2)) # Δ_dec * ones(ae.N)
ΔW_enc = momentum * ΔW_enc + learning_rate * ∂W_enc + regularization * ae.W_enc
Δb_enc = momentum * Δb_enc + learning_rate * ∂b_enc
ΔW_dec = momentum * ΔW_dec + learning_rate * ∂W_dec + regularization * ae.W_dec
Δb_dec = momentum * Δb_dec + learning_rate * ∂b_dec
ae.W_enc -= ΔW_enc
ae.b_enc -= Δb_enc
ae.W_dec -= ΔW_dec
ae.b_dec -= Δb_dec
@show l(X)
end
end
function test()
X, y = MNIST.traindata()
X = convert(Array{Float32}, X)
X = flatten(X)
# X, y = subset(X, y, [5, 2])
X, y = subset(X, y, 5000)
encoder = AutoEncoder{Float32}(100)
train!(encoder, X)
# feats = reshape(encoder.W_enc', 28, 28, 100)
# heatmap(flipdim(reshape(permutedims(reshape(feats, 28, 28, 10, 10), (2, 3, 1, 4)), 280, 280), 1))
encoder
end
encoder = test()
using MXNet
using Plots
plotlyjs()
#--------------------------------------------------------------------------------
# extract parameters and visualize trained model
# helper function for wisualizing weights
include("visualize_mnist.jl")
# data provider
include("data_provider_mnist.jl")
train_provider, eval_provider = get_mnist_providers(data_name=:data, label_name=:output_label, subset=[0, 1, 7])
# load pre-trained model
prefix = "ae_encoder_mnist"
epochs = 1000
model = cd("$(dirname(@__FILE__))/output/models") do
mx.load_checkpoint(prefix, epochs, MXNet.mx.FeedForward)
end
# predict model
eval_labels = vec(convert(Array{Int64}, eval_provider.label_arrays[1]))
predictions = mx.predict(model, eval_provider)
W = copy(model.arg_params[:enc_0_1_weight])
# plot
cd("$(dirname(@__FILE__))/output/plots") do
p1 = scatter(predictions[1,:], predictions[2,:], group=eval_labels)
savefig(p1, "$(prefix)_prediction_scatter.pdf")
p2 = mnist_visualize(W)
savefig(p2, "$(prefix)_weights.pdf")
end
using MXNet
using Plots
plotlyjs()
#--------------------------------------------------------------------------------
# extract parameters and visualize trained model
# helper function for wisualizing weights
include("visualize_mnist.jl")
# data provider
include("data_provider_mnist.jl")
train_provider, eval_provider = get_mnist_providers(data_name=:data, label_name=:output_label, subset=[0, 1, 7])
# load pre-trained model
prefix = "ae_full_mnist"
epochs = 1000
model = cd("$(dirname(@__FILE__))/output/models") do
mx.load_checkpoint(prefix, epochs, MXNet.mx.FeedForward)
end
# predict model
eval_data = eval_provider.data_arrays[1]
predictions = mx.predict(model, eval_provider)
# plot
cd("$(dirname(@__FILE__))/output/plots") do
p1 = mnist_visualize(predictions[:,1:100])
savefig(p1, "$(prefix)_predictions.pdf")
p2 = mnist_visualize(eval_data[:,1:100])
savefig(p2, "$(prefix)_original.pdf")
end
using MXNet
#--------------------------------------------------------------------------------
# define and train autoencoder
# autoencoder factory functions
include("autoencoder.jl")
# data provider
include("data_provider_mnist.jl")
batch_size = 100
train_provider, eval_provider = get_mnist_autoencoder_providers(batch_size, data_name=:data, label_name=:output_label, subset=[0, 1, 7])
# build the autoencoder
layer_dims = [784, 64, 256, 64, 2]
ae, encoder = autoencoder(layer_dims, data_name=:data, label_name=:output_label)
# setup model
model = mx.FeedForward(ae, context=mx.gpu.(0))
# optimization algorithm
optimizer = mx.ADAM()
# optimizer = mx.SGD(lr=0.1, momentum=0.9, weight_decay=0.00001)
# fit parameters
mx.fit(model, optimizer, train_provider, n_epoch=1000, eval_metric=mx.MSE())
# create the encoder partial model
encoder_model = mx.FeedForward(encoder)
encoder_model.arg_params = filter((k,v) -> !startswith(string(k), "dec"), model.arg_params)
encoder_model.aux_params = model.aux_params
ae_prefix = "ae_full_mnist"
encoder_prefix = "ae_encoder_mnist"
# save models
cd("$(dirname(@__FILE__))/output/models") do
mx.save_checkpoint(model, ae_prefix, optimizer.state)
mx.save_checkpoint(encoder_model, encoder_prefix, optimizer.state)
end
# save visualizations of the network
cd("$(dirname(@__FILE__))/output/plots") do
write("$ae_prefix.dot", mx.to_graphviz(ae))
run(`dot -Tpdf $ae_prefix.dot -o $ae_prefix.pdf`)
run(`rm $ae_prefix.dot`)
write("$encoder_prefix.dot", mx.to_graphviz(encoder))
run(`dot -Tpdf $encoder_prefix.dot -o $encoder_prefix.pdf`)
run(`rm $encoder_prefix.dot`)
end
# autoencoder factory functions
# factory for single layer
function build_layer(input, num_hidden, weight, name, act)
fc = mx.FullyConnected(input, weight, num_hidden=num_hidden, name="$(name)_fc")
if act != :none
fc = mx.Activation(fc, act_type=act, name="$(name)_act")
end
# fc = mx.BatchNorm(data=fc)
return fc
end
# recursive factory for autoencoder
function build_autoencoder(input, layer_dims; inner_activation="relu", outer_activation="sigmoid", depth=0)
n = length(layer_dims)
if n < 2
return input, input
end
if n == 2
inner_activation = :none
end
name = "$(depth)_$(depth+1)"
W = mx.Variable("enc_$(name)_weight")
Wt = W'
mx.set_attr(Wt, :name, "enc_$(name)_weight")
encoder_input = build_layer(input, layer_dims[2], W, "enc_$(depth)_$(depth+1)", inner_activation)
decoder_input, encoder = build_autoencoder(encoder_input, layer_dims[2:end];
inner_activation=inner_activation,
outer_activation=inner_activation,
depth=depth+1)
decoder = build_layer(decoder_input, layer_dims[1], Wt, "dec_$(depth+1)_$(depth)", outer_activation)
return decoder, encoder
end
function autoencoder(layer_dims; data_name="data", label_name="output_label")
data = mx.Variable(data_name)
label = mx.Variable(label_name)
autoencoder, encoder = build_autoencoder(data, layer_dims)
cost = mx.LinearRegressionOutput(autoencoder, label, name="cost")
return cost, encoder
end
using MNIST
function get_mnist_autoencoder_providers(batch_size::Int; data_name=:data, label_name=:softmax_label, subset=0:9)
train_data, train_label = MNIST.traindata()
idx = BitArray(indexin(train_label, subset))
train_data = train_data[:,idx]
# train_label = train_data
scale!(train_data, 1/255)
train_provider = mx.ArrayDataProvider(data_name=>train_data, label_name=>train_data, batch_size=batch_size, shuffle=true)
eval_data, eval_label = MNIST.testdata()
idx = BitArray(indexin(eval_label, subset))
eval_data = eval_data[:,idx]
# eval_label = eval_data
scale!(eval_data, 1/255)
eval_provider = mx.ArrayDataProvider(data_name=>eval_data, label_name=>eval_data, batch_size=batch_size, shuffle=false)
return train_provider, eval_provider
end
function get_mnist_providers(batch_size::Int=0; data_name=:data, label_name=:softmax_label, subset=0:9)
train_data, train_label = MNIST.traindata()
idx = BitArray(indexin(train_label, subset))
train_data = train_data[:,idx]
train_label = train_label[idx]
scale!(train_data, 1/255)
train_provider = mx.ArrayDataProvider(data_name=>train_data, label_name=>train_label, batch_size=batch_size, shuffle=true)
eval_data, eval_label = MNIST.testdata()
idx = BitArray(indexin(eval_label, subset))
eval_data = eval_data[:,idx]
eval_label = eval_label[idx]
scale!(eval_data, 1/255)
eval_provider = mx.ArrayDataProvider(data_name=>eval_data, label_name=>eval_label, batch_size=batch_size, shuffle=false)
return train_provider, eval_provider
end
{
"nodes": [
{
"op": "null",
"name": "data",
"inputs": []
},
{
"op": "null",
"name": "enc_0_1_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_0_1_fc_bias",
"attr": {"num_hidden": "64"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_0_1_fc",
"attr": {"num_hidden": "64"},
"inputs": [[0, 0, 0], [1, 0, 0], [2, 0, 0]]
},
{
"op": "Activation",
"name": "enc_0_1_act",
"attr": {"act_type": "relu"},
"inputs": [[3, 0, 0]]
},
{
"op": "null",
"name": "enc_1_2_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_1_2_fc_bias",
"attr": {"num_hidden": "256"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_1_2_fc",
"attr": {"num_hidden": "256"},
"inputs": [[4, 0, 0], [5, 0, 0], [6, 0, 0]]
},
{
"op": "Activation",
"name": "enc_1_2_act",
"attr": {"act_type": "relu"},
"inputs": [[7, 0, 0]]
},
{
"op": "null",
"name": "enc_2_3_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_2_3_fc_bias",
"attr": {"num_hidden": "64"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_2_3_fc",
"attr": {"num_hidden": "64"},
"inputs": [[8, 0, 0], [9, 0, 0], [10, 0, 0]]
},
{
"op": "Activation",
"name": "enc_2_3_act",
"attr": {"act_type": "relu"},
"inputs": [[11, 0, 0]]
},
{
"op": "null",
"name": "enc_3_4_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_3_4_fc_bias",
"attr": {"num_hidden": "2"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_3_4_fc",
"attr": {"num_hidden": "2"},
"inputs": [[12, 0, 0], [13, 0, 0], [14, 0, 0]]
}
],
"arg_nodes": [0, 1, 2, 5, 6, 9, 10, 13, 14],
"node_row_ptr": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16
],
"heads": [[15, 0, 0]],
"attrs": {"mxnet_version": ["int", 903]}
}
\ No newline at end of file
{
"nodes": [
{
"op": "null",
"name": "data",
"inputs": []
},
{
"op": "null",
"name": "enc_0_1_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_0_1_fc_bias",
"attr": {"num_hidden": "64"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_0_1_fc",
"attr": {"num_hidden": "64"},
"inputs": [[0, 0, 0], [1, 0, 0], [2, 0, 0]]
},
{
"op": "Activation",
"name": "enc_0_1_act",
"attr": {"act_type": "relu"},
"inputs": [[3, 0, 0]]
},
{
"op": "null",
"name": "enc_1_2_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_1_2_fc_bias",
"attr": {"num_hidden": "256"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_1_2_fc",
"attr": {"num_hidden": "256"},
"inputs": [[4, 0, 0], [5, 0, 0], [6, 0, 0]]
},
{
"op": "Activation",
"name": "enc_1_2_act",
"attr": {"act_type": "relu"},
"inputs": [[7, 0, 0]]
},
{
"op": "null",
"name": "enc_2_3_weight",
"inputs": []
},
{
"op": "null",
"name": "enc_2_3_fc_bias",
"attr": {"num_hidden": "64"},
"inputs": []
},
{
"op": "FullyConnected",
"name": "enc_2_3_fc",
"attr": {"num_hidden": "64"},
"inputs": [[8, 0, 0], [9, 0, 0], [10, 0, 0]]
},
{
"op": "Activation",
"name": "enc_2_3_act",
"attr": {"act_type": "relu"},
"inputs": [[11, 0, 0]]
},
{
"op": "null",
"name": "enc_3_4_weight",
"inputs": []
},
{
"op": "null",