8/7/2020 autoencoder_mpl_basic.

ipynb - Colaboratory

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/
Instructions for updating:
non-resource variables are not supported in the long term

from google.colab import drive

drive.mount('/content/drive')

Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=

Enter your authorization code:

··········
Mounted at /content/drive

import numpy as np
import math
import numpy as np
from os.path import join, exists, basename, split
import json
import shutil
import scipy.io

class HypImg:

def __init__( self , spectralInput , labels=None, wavelengths=None, bands=None ):

# if input is of shape [numRows x numCols x numBands], convert to [numSamples x

if len( spectralInput.shape ) == 3:
self.numRows , self.numCols , self.numBands = spectralInput.shape
self.numSamples = self.numRows * self.numCols
self.spectra = (np.reshape( spectralInput , ( -1, self.numBands ) )).astype
self.spectraCube = spectralInput.astype(np.float)
else:
self.numSamples , self.numBands = spectralInput.shape
self.numRows = None
self.numCols = None
self.spectra = spectralInput.astype(np.float)
self.spectraCube = None

# if labels provided, determine number of classes and one-hot labels

if labels is not None:
if len(labels.shape) == 2:
self.labels = np.reshape(labels, -1)
else:
self.labels = labels
self.numClasses = len( np.unique(self.labels)[np.unique(self.labels)>0] )
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 1/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# create one-hot labels for classes > 0

self.labelsOnehot = np.zeros((self.numSamples, self.numClasses))
self.labelsOnehot[np.arange(self.numSamples)[self.labels>0], (self.labels-1

self.labels = self.labels[:,np.newaxis]
else:
self.labels = None
self.labelsOnehot = None
self.numClasses = None

self.wavelengths = wavelengths
self.bands = bands

def pre_process( self , method='minmax' ):

"""Pre-process data for input into the network. Stores in the spectraPrep attr
Args:
method (str): Method of pre-processing. Current options: 'minmax'
"""
if method == 'minmax':
# scales each spectra to be between [0 1] (lower bound is actually a small
self.spectraPrep = self.spectra - np.transpose(np.tile(np.min(self.spectra
self.spectraPrep = self.spectraPrep / np.transpose(np.tile(np.max(self.spe

class Iterator:

def __init__(self, dataSamples,targets,batchSize=None):

self.dataSamples = dataSamples
self.targets = targets
self.numSamples = np.shape(dataSamples)[0]
if batchSize is not None:
self.batchSize = batchSize
else:
self.batchSize = self.numSamples
self.currentBatch = np.arange(self.batchSize)

def next_batch(self):

batchData = self.dataSamples[self.currentBatch, :]
batchTargets = self.targets[self.currentBatch, :]

# update current batch

self.currentBatch += self.batchSize
self.currentBatch[self.currentBatch >= self.numSamples] = \
self.currentBatch[self.currentBatch >= self.numSamples] - self.numSamples

return batchData , batchTargets

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 2/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

def get_batch(self, idx):

batchData = self.dataSamples[idx, :]
batchTargets = self.targets[idx, :]

return batchData, batchTargets

def reset_batch(self):
""" Resets the current batch to the beginning.
"""

self.currentBatch = np.arange(self.batchSize)

def shuffle(self):
""" Randomly permutes all dataSamples (and corresponding targets).
"""
idx = np.random.permutation(np.shape(self.dataSamples)[0])
self.dataSamples = self.dataSamples[idx,:]
self.targets = self.targets[idx,:]

def create_variable(shape,method='gaussian',wd=False):

return tf.Variable(init_weight(method, shape, wd=wd))

def layer_fullyConn(input, W, b):

return tf.matmul(input, W) + b

def layer_conv1d(input, W, b, stride=1,padding='SAME'):

if (padding!='SAME')&(padding!='VALID'):
raise ValueError('unknown padding type: %s. Use SAME or VALID' % padding)
if stride < 1:
raise ValueError('stride must be greater than 0. Stride = %d found in conv laye

return tf.nn.conv1d(input,W,stride=stride,padding=padding) + b

def layer_deconv1d(input, W, b, outputShape, stride=1,padding='SAME'):

if (padding!='SAME')&(padding!='VALID'):
raise ValueError('unknown padding type: %s. Use SAME or VALID' % padding)
if stride < 1:
raise ValueError('stride must be greater than 0. Stride = %d found in deconv la

return tf.nn.conv1d_transpose(input,W,outputShape,strides=stride,padding=padding) +

def layer_activation(input, func='sigmoid'):

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 3/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

if func == 'relu':
a = tf.nn.relu(input)
elif func == 'sigmoid':
a = tf.nn.sigmoid(input)
elif func == 'linear':
a = input
else:
raise ValueError('unknown activation function: %s. Use relu, sigmoid or linear

return a

def conv_output_shape(inputShape, filterSize, padding, stride):

if padding=='VALID':
outputShape = np.ceil( (inputShape - (filterSize-1))/stride )
elif padding=='SAME':
outputShape = np.ceil(inputShape / stride)
else:
raise ValueError('unknown padding type: %s. Use SAME or VALID' % padding)

return int(outputShape)

def train_step(loss, learning_rate=1e-3, decay_steps=None, decay_rate=None, piecewise_

method='Adam'):

global_step = tf.Variable(0, trainable=False, name='global_step')

# update learning rate for current step

if decay_rate != None:
lr = tf.train.exponential_decay(learning_rate,
global_step,
decay_steps,
decay_rate, staircase=True)
elif piecewise_bounds != None:
lr = tf.train.piecewise_constant(global_step, piecewise_bounds, [learning_rate
else:
lr = learning_rate

if method == 'Adam':
optimizer = tf.train.AdamOptimizer(lr)
elif method == 'SGD':
optimizer = tf.train.GradientDescentOptimizer(lr)
else:
raise ValueError('unknown optimisation method: %s. Use Adam or SGD.' % method)

train_op = optimizer.minimize(loss, global_step=global_step)

return train_op

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 4/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

def loss_function_reconstruction_1D(y_reconstructed,y_target,func='SSE'):

if func == 'SSE':
# sum of squared errors loss
loss = tf.reduce_sum( tf.square(y_target - y_reconstructed) )

elif func == 'CSA':

# cosine of spectral angle loss
normalize_r = tf.math.l2_normalize(tf.transpose(y_reconstructed),axis=0)
normalize_t = tf.math.l2_normalize(tf.transpose(y_target),axis=0)
loss = tf.reduce_sum( 1 - tf.reduce_sum(tf.multiply(normalize_r, normalize_t),a

elif func == 'SA':

# spectral angle loss
normalize_r = tf.math.l2_normalize(tf.transpose(y_reconstructed),axis=0)
normalize_t = tf.math.l2_normalize(tf.transpose(y_target),axis=0)
loss = tf.reduce_sum( tf.math.acos(tf.reduce_sum(tf.multiply(normalize_r, norma

elif func == 'SID':

# spectral information divergence loss
t = tf.divide( tf.transpose(y_target) , tf.reduce_sum(tf.transpose(y_target),ax
r = tf.divide( tf.transpose(y_reconstructed) , tf.reduce_sum(tf.transpose(y_re
loss = tf.reduce_sum( tf.reduce_sum( tf.multiply(t,tf.log(tf.divide(t,r))) , ax
+ tf.reduce_sum( tf.multiply(r,tf.log(tf.divide(r,t))) ,
else:
raise ValueError('unknown loss function: %s. Use SSE, CSA, SA or SID.' % func)

return loss

def loss_function_crossentropy_1D( y_pred, y_target, class_weights=None, num_classes=No

if class_weights==None:
class_weights = tf.constant(1,shape=[num_classes],dtype=tf.dtypes.float32)

sample_weights = tf.reduce_sum( tf.multiply(y_target, class_weights ), axis=1) # we

loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(
onehot_labels=y_target,logits=y_pred,weights=sample_weights ) )

return loss

def loss_weight_decay(wdLambda):

return tf.multiply( wdLambda , tf.reduce_sum(tf.get_collection('wd')) )

def balance_classes(y_target,num_classes):

y_target = tf.reshape( y_target, [-1, num_classes] )

class_count = tf.add( tf.reduce_sum( y_target, axis=0 ), tf.constant( [1]*num_clas
class weights tf multiply( tf divide( tf ones( ( 1 num classes) ) class count
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 5/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
class_weights = tf.multiply( tf.divide( tf.ones( ( 1, num_classes) ), class_count

return class_weights

def save_model(addr,sess,saver,current_epoch,epochs_to_save):

if current_epoch in epochs_to_save:
saver.save(sess, join(addr,"epoch_%i"%(current_epoch),"model.ckpt"))

def load_model(addr,sess):
"""Loads a model from the address of a checkpoint.
Args:
addr (str): Address of a directory to save checkpoint for current epoch.
sess (obj): Tensor flow session object.
"""
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, join(addr, 'model.ckpt'))

def save_config(net_obj,addr):
"""Saves a network config file. Saves the variables listed in net_config within the
Args:
net_obj (obj): Network object.
addr (obj): Directory of where to store the config.json file.
"""

data = {}
for config_parameter in net_obj.net_config:
data[config_parameter] = getattr(net_obj,config_parameter)

with open(join(addr,'config.json'), 'w') as outfile:

json.dump(data, outfile)

def load_config(net_obj,addr):
"""Loads a network config file. Loads from variables in the config.json file and ov
object. Applies to variables in the net_config list in the network object.
Args:
net_obj (obj): Network object.
addr (obj): Directory location of config.json file.
"""

with open(addr, 'r') as outfile:

data = json.load(outfile)

for config_parameter in data:

setattr(net_obj,config_parameter,data[config_parameter])

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 6/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

def train( net_obj , dataTrain, dataVal, train_op_name, n_epochs, save_addr, visualise

save_epochs=[1000] ):

if np.shape(dataTrain.dataSamples)[1] != net_obj.inputSize:
raise Exception('the data dimensionality must match the network input size. '
'Data size: %d, network input size: %d'%(np.shape(dataTrain.dat

batchSize = dataTrain.batchSize
numSamples = dataTrain.numSamples

numIters = numSamples // batchSize

if (numSamples % batchSize)>0:
numIters+=1

with tf.Session() as sess:

sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()

# check if addr has 'epoch' in name and contains a checkpoint

if exists(join(save_addr,'checkpoint')) & ('epoch' in basename(save_addr)):
# load a checkpoint
saver.restore(sess,join(save_addr,'model.ckpt'))
epoch_start = int((basename(save_addr)).split('_')[-1]) + 1
save_addr = split(save_addr)[0]
else:
# save directory is empty
epoch_start = 0
# create network config file in directory
save_config(net_obj,save_addr)

for epoch_i in range(epoch_start, n_epochs+1):

train_error = []
for batch_i in range(numIters):
train_batch_x , train_batch_y = dataTrain.next_batch()

# update weights and biases

sess.run(net_obj.train_ops['%s_train'%(train_op_name)], feed_dict={net_
net_

# training loss
if visualiseRateTrain > 0:
if epoch_i % visualiseRateTrain == 0:
train_error.append( net_obj.train_ops['%s_loss' % (train_op_nam
{net_obj.x: train_batch_x, net_obj.y_target: train_batch_y

if batch_i == numIters - 1:
dataTrain.reset_batch()

# outputs average batch error

if visualiseRateTrain > 0:
if epoch_i % visualiseRateTrain == 0:
train_error = np.array(train_error)
print("epoch: %d, training loss: %g" % (epoch_i, np.mean(train_erro
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 7/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# iterate over validation samples and output loss

if visualiseRateVal > 0:
if epoch_i % visualiseRateVal == 0:

val_error = []
for batch_i in range(dataVal.numSamples // dataVal.batchSize):
val_batch_x, val_batch_y = dataVal.next_batch()

val_error.append( net_obj.train_ops['%s_loss' % (train_op_name

{net_obj.x: val_batch_x, net_obj.y_target: val_batch_y

if batch_i == (dataVal.numSamples // dataVal.batchSize)-1:

dataVal.reset_batch()

val_error = np.array(val_error)
print("epoch: %d, validation loss: %g" % (epoch_i, np.mean(val_erro

save_model(save_addr,sess,saver,epoch_i,save_epochs)

def init_weight(opts, shape, stddev=0.1, const=0.1, wd = False, dtype=tf.float32):

if opts == 'gaussian':
weights = tf.random_normal(shape, stddev=stddev, dtype=dtype)
elif opts == 'truncated_normal':
weights = tf.truncated_normal(shape, stddev=stddev)
elif opts == 'xavier':
h = shape[0]
w = shape[1]
try:
num_in = shape[2]
except:
num_in = 1
sc = math.sqrt(3.0 / (h * w * num_in))
weights = tf.multiply(tf.random_normal(shape, dtype=dtype) * 2 - 1, sc)
elif opts == 'xavier_improved':
h = shape[0]
w = shape[1]
try:
num_out = shape[3]
except:
num_out = 1
sc = math.sqrt(2.0 / (h * w * num_out))
weights = tf.multiply(tf.random_normal(shape, dtype=dtype), sc)
elif opts == 'constant':
weights = tf.constant(const, shape)
else:
raise ValueError('Unknown weight initialization method %s' % opts)

# set up weight decay on weights

if wd:
weight decay = tf nn l2 loss(weights)
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 8/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
weight_decay = tf.nn.l2_loss(weights)
tf.add_to_collection('wd', weight_decay)

return weights

class mlp_1D_network():

def __init__( self , configFile=None, inputSize=None , encoderSize=[50,30,10] , act

tiedWeights=None , weightInitOpt='truncated_normal' , weightStd=0.1,
activationFuncFinal='linear' ):

self.inputSize = inputSize
self.activationFunc = activationFunc
self.tiedWeights = tiedWeights
self.skipConnect = skipConnect
self.weightInitOpt = weightInitOpt
self.weightStd = weightStd
self.encodersize = encoderSize
self.activationFuncFinal = activationFuncFinal

self.net_config = ['inputSize','encodersize','activationFunc','tiedWeights','we
'skipConnect','activationFuncFinal']
# loading config file overwrites input arguments
if configFile is not None:
load_config(self,configFile)

if self.inputSize is None:
raise Exception('value must be given for inputSize (not None)')

self.encoderSize = [self.inputSize] + self.encodersize

self.decoderSize = self.encoderSize[::-1]

self.x = tf.placeholder("float", [None, self.inputSize])

self.y_target = tf.placeholder("float", [None, self.inputSize])

self.weights = { }
self.biases = { }
self.h = {}
self.a = {}
self.train_ops = {}
self.modelsAddrs = {}

if self.tiedWeights is None:
self.tiedWeights = [0]*(len(self.encoderSize)-1)

for layerNum in range( len( self.encoderSize ) - 1 ):

self.weights['encoder_w%i'%(layerNum+1)] = \
create_variable([self.encoderSize[layerNum],
self.encoderSize[layerNum+1]],self.weightInitO

# decoder weights
for layerNum in range( len( self.decoderSize ) - 1 ):
if self.tiedWeights[layerNum] == 0:
self.weights['decoder_w%i' % (len( self.encoderSize ) + layerNum )] =
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 9/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
g [ _ ( ( ) y )]
create_variable([self.decoderSize[layerNum], self.decoderSize[layerN
self.weightInitOpt, wd=True)
elif self.tiedWeights[layerNum] == 1:
self.weights['decoder_w%i' % (len(self.encoderSize) + layerNum)] = \
tf.transpose( self.weights['encoder_w%i'%(len(self.encoderSize)-1-
else:
raise ValueError('unknown tiedWeights value: %i. '
'Must be 0 or 1 for each layer (or None).' % tiedWeig

# encoder biases
for layerNum in range( len( self.encoderSize ) - 1 ):
self.biases['encoder_b%i'%(layerNum+1)] = \
create_variable([self.encoderSize[layerNum+1]] , self.weightInitOpt, wd=

# decoder biases
for layerNum in range( len( self.decoderSize ) - 1 ):
self.biases['decoder_b%i' % (len( self.encoderSize ) + layerNum )] = \
create_variable([self.decoderSize[layerNum + 1]], self.weightInitOpt, w

# build network using encoder, decoder and x placeholder as input

# build encoder
self.a['a0'] = self.x
for layerNum in range( 1 , len( self.encoderSize ) ):
self.h['h%d' % (layerNum)] = \
layer_fullyConn(self.a['a%d'%(layerNum-1)], self.weights['encoder_w%d'%
self.biases['encoder_b%d'%(layerNum)])
self.a['a%d' % (layerNum)] = layer_activation(self.h['h%d' % (layerNum)],

# latent representation
self.z = self.a['a%d' % (layerNum)]

# build decoder
for layerNum in range( 1 , len( self.decoderSize ) ):
absLayerNum = len(self.encoderSize) + layerNum - 1
self.h['h%d' % (absLayerNum)] = \
layer_fullyConn(self.a['a%d'%(absLayerNum-1)], self.weights['decoder_w%
self.biases['decoder_b%d'%(absLayerNum)])
if layerNum < len( self.decoderSize )-1:
if self.skipConnect:
self.h['h%d' % (absLayerNum)] += self.h['h%d' % (len(self.decoderS
self.a['a%d' % (absLayerNum)] = \
layer_activation(self.h['h%d' % (absLayerNum)], self.activationFun
else:
if self.skipConnect:
self.h['h%d' % (absLayerNum)] += self.a['a0']
self.a['a%d' % (absLayerNum)] = \
layer_activation(self.h['h%d' % (absLayerNum)], self.activationFun

# output of final layer

self.y_recon = self.a['a%d' % (absLayerNum)]

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 10/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

def add_train_op(self,name,lossFunc='CSA',learning_rate=1e-3, decay_steps=None, de

piecewise_bounds=None, piecewise_values=None, method='Adam', wd_la

# construct loss op
self.train_ops['%s_loss'%name] = loss_function_reconstruction_1D(self.y_recon,

# weight decay loss contribution

wdLoss = loss_weight_decay(wd_lambda)

# construct training op
self.train_ops['%s_train'%name] = \
train_step(self.train_ops['%s_loss'%name]+wdLoss, learning_rate, decay_ste
piecewise_bounds, piecewise_values,method)

def train(self, dataTrain, dataVal, train_op_name, n_epochs, save_addr, visualiseRa

save_epochs=[1000]):

# make sure a checkpoint is saved at n_epochs

if n_epochs not in save_epochs:
save_epochs.append(n_epochs)

train( self, dataTrain, dataVal, train_op_name, n_epochs, save_addr, visualise

visualiseRateVal, save_epochs )

def add_model(self,addr,modelName):

self.modelsAddrs[modelName] = addr

def encoder( self, modelName, dataSamples ):

with tf.Session() as sess:

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get latent values

dataZ = sess.run(self.z, feed_dict={self.x: dataSamples})

return dataZ

def decoder( self, modelName, dataZ ):

with tf.Session() as sess:

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 11/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get reconstruction
dataY_recon = sess.run(self.y_recon, feed_dict={self.z: dataZ})

return dataY_recon

def encoder_decoder( self, modelName, dataSamples ):

with tf.Session() as sess:

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get reconstruction
dataY_recon = sess.run(self.y_recon, feed_dict={self.x: dataSamples})

return dataY_recon

class cnn_1D_network():

def __init__( self , configFile=None, inputSize=None , zDim=5, encoderNumFilters=[1

encoderFilterSize=[20,10,10], activationFunc='sigmoid', tiedWeights=N
weightInitOpt='truncated_normal', weightStd=0.1, skipConnect=False,
encoderStride=[1,1,1], activationFuncFinal='linear' ):

self.inputSize = inputSize
self.tiedWeights = tiedWeights
self.skipConnect = skipConnect
self.weightInitOpt = weightInitOpt
self.weightStd = weightStd
self.zDim = zDim
self.padding = padding
self.activationFunc = activationFunc
self.encoderStride = encoderStride
self.encoderNumfilters = encoderNumFilters
self.encoderFiltersize = encoderFilterSize
self.activationFuncFinal = activationFuncFinal

self.net_config = ['inputSize','zDim','encoderNumfilters','encoderFiltersize',
'weightInitOpt','weightStd','skipConnect','padding','encode
# loading config file overwrites input arguments
if configFile is not None:
load_config(self,configFile)

if self.inputSize is None:
raise Exception('value must be given for inputSize (not None)')

if not (len(self.encoderFiltersize) == len(self.encoderNumfilters) == len(self

raise Exception('the length of encoderNumfilters, encoderFilterSize and en
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 12/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

self.encoderNumFilters = [1] + self.encoderNumfilters

self.decoderNumFilters = self.encoderNumFilters[::-1]
self.encoderFilterSize = self.encoderFiltersize
self.decoderFilterSize = self.encoderFiltersize[::-1]
self.decoderStride = encoderStride[::-1]

self.x = tf.placeholder("float", [None, self.inputSize])

self.y_target = tf.placeholder("float", [None, self.inputSize])

self.weights = { }
self.biases = { }
self.h = {}
self.a = {}
self.train_ops = {}
self.modelsAddrs = {}

if self.tiedWeights is None:
self.tiedWeights = [0]*(len(self.encoderNumFilters)-1)

# pre-compute shape of data after each layer

self.encoderDataShape = [self.inputSize]
for layerNum in range( len( self.encoderNumFilters ) - 1 ):
self.encoderDataShape.append(conv_output_shape(
self.encoderDataShape[layerNum],self.encoderFilterSize[layerNum],self.
self.encoderDataShape.append(self.zDim)
self.encoderDataShape[layerNum + 1] = self.encoderDataShape[-2] * self.encoderN
self.decoderDataShape = self.encoderDataShape[::-1]

#--

# encoder weights
for layerNum in range( len( self.encoderNumFilters ) - 1 ):
self.weights['encoder_w%i'%(layerNum+1)] = \
create_variable([self.encoderFilterSize[layerNum], self.encoderNumFilte
self.encoderNumFilters[layerNum+1]],weightInit
self.weights['encoder_w%i' % (layerNum + 2)] = create_variable(
[self.encoderDataShape[layerNum+1], self.zDim],self.weightInitOpt, wd=True

# decoder weights
self.weights['decoder_w%i' % (layerNum + 3)] = create_variable(
[self.zDim,self.decoderDataShape[1]], self.weightInitOpt, wd=True)
for layerNum in range( len( self.decoderNumFilters ) - 1 ):
if self.tiedWeights[layerNum] == 0:
self.weights['decoder_w%i' % (len( self.encoderDataShape ) + layerNum +
create_variable([self.decoderFilterSize[layerNum], self.decoderNumF
self.decoderNumFilters[layerNum]], self.we
elif self.tiedWeights[layerNum] == 1:
self.weights['decoder_w%i' % (len( self.encoderNumFilters )+layerNum+2
self.weights['encoder_w%i' % (len(self.encoderNumFilters)-1 - laye
else:
raise ValueError('unknown tiedWeights value: %i. '
'Must be 0 or 1 for each layer (or None).' % tiedWeig
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 13/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# encoder biases
for layerNum in range( len( self.encoderNumFilters ) - 1 ):
self.biases['encoder_b%i'%(layerNum+1)] = \
create_variable([self.encoderNumFilters[layerNum+1]] , self.weightInitO
self.biases['encoder_b%i'%(layerNum+2)] = create_variable([self.zDim] , self.we

# decoder biases
self.biases['decoder_b%i' % (layerNum + 3)] = \
create_variable([self.decoderDataShape[1]], self.weightInitOpt, wd=True)
for layerNum in range( len( self.decoderNumFilters ) - 1 ):
self.biases['decoder_b%i' % (len( self.encoderDataShape ) + layerNum + 1)]
create_variable([self.decoderNumFilters[layerNum+1]], self.weightInitOpt

# build network using encoder, decoder and x placeholder as input

# build encoder
self.a['a0'] = tf.expand_dims(self.x,axis=2) # expand to shape None x inputS
for layerNum in range( 1 , len( self.encoderNumFilters ) ):
self.h['h%d' % (layerNum)] = \
layer_conv1d(self.a['a%d'%(layerNum-1)], self.weights['encoder_w%d'%(la
self.biases['encoder_b%d'%(layerNum)],padding=self
self.a['a%d' % (layerNum)] = layer_activation(self.h['h%d' % (layerNum)],
self.a['a%d'%(layerNum)] = tf.reshape( self.a['a%d'%(layerNum)], [-1,self.encod
self.h['h%d' % (layerNum+1)] = \
layer_fullyConn(
self.a['a%d'%(layerNum)],self.weights['encoder_w%d'%(layerNum+1)],self
self.a['a%d' % (layerNum+1)] = layer_activation(self.h['h%d' % (layerNum+1)],

# latent representation
self.z = self.a['a%d' % (layerNum+1)] # collapse a dim

# build decoder
self.h['h%d' % (layerNum+2)] = \
layer_fullyConn(self.a['a%d' % (layerNum+1)],
self.weights['decoder_w%d' % (layerNum+2)],self.bia
if skipConnect:
self.h['h%d' % (layerNum+2)] += tf.reshape(
self.h['h%d' % (len( self.decoderNumFilters ) - 1)] , [-1,self.encoderD
self.a['a%d' % (layerNum+2)] = layer_activation(self.h['h%d' % (layerNum+2)],
self.a['a%d' % (layerNum + 2)] = tf.reshape(
self.a['a%d' % (layerNum + 2)], [-1,int(self.decoderDataShape[1]/self.encod
for layerNum in range( 1 , len( self.decoderNumFilters ) ):
absLayerNum = len( self.encoderDataShape ) + layerNum
outputShape = [tf.shape(self.a['a%d' % (absLayerNum-1)] )[0],
self.decoderDataShape[layerNum+1],self.decoderNumFilters[lay
self.h['h%d' % (absLayerNum)] = \
layer_deconv1d(self.a['a%d'%(absLayerNum-1)], self.weights['decoder_w%d
self.biases['decoder_b%d'%(absLayerNum)],
outputShape, padding=self.padding, stride=self.d
if layerNum < len( self.decoderNumFilters )-1:
if self.skipConnect:
self.h['h%d' % (absLayerNum)] += self.h['h%d' % (len( self.decoderN
self a['a%d' % (absLayerNum)] layer activation(self h['h%d' % (absLay
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 14/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
self.a[ a%d % (absLayerNum)] = layer_activation(self.h[ h%d % (absLay
else:
if self.skipConnect:
self.h['h%d' % (absLayerNum)] += self.a['a0']
self.a['a%d' % (absLayerNum)] = layer_activation(self.h['h%d' % (absLay

# output of final layer

self.y_recon = tf.squeeze( self.a['a%d' % (absLayerNum)] , axis=2)

def add_train_op(self,name,lossFunc='SSE',learning_rate=1e-3, decay_steps=None, de

piecewise_bounds=None, piecewise_values=None, method='Adam', wd_la

# construct loss op
self.train_ops['%s_loss'%name] = loss_function_reconstruction_1D(self.y_recon,

# weight decay loss contribution

wdLoss = loss_weight_decay(wd_lambda)

# construct training op
self.train_ops['%s_train'%name] = \
train_step(self.train_ops['%s_loss'%name]+wdLoss, learning_rate, decay_ste

def train(self, dataTrain, dataVal, train_op_name, n_epochs, save_addr, visualiseRa

save_epochs=[1000]):

# make sure a checkpoint is saved at n_epochs

if n_epochs not in save_epochs:
save_epochs.append(n_epochs)

train( self, dataTrain, dataVal, train_op_name, n_epochs, save_addr, visualise

def add_model(self,addr,modelName):

self.modelsAddrs[modelName] = addr

def encoder( self, modelName, dataSamples ):

with tf.Session() as sess:

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get latent values

dataZ = sess.run(self.z, feed_dict={self.x: dataSamples})

return dataZ
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 15/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
return dataZ

def decoder( self, modelName, dataZ ):

with tf.Session() as sess:

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get reconstruction
dataY_recon = sess.run(self.y_recon, feed_dict={self.z: dataZ})

return dataY_recon

def encoder_decoder( self, modelName, dataSamples ):

with tf.Session() as sess:

# load the model

load_model(self.modelsAddrs[modelName], sess)

# get reconstruction
dataY_recon = sess.run(self.y_recon, feed_dict={self.x: dataSamples})

return dataY_recon

import sys
import time

def reporthook(count, block_size, total_size):

global start_time
if count == 0:
start_time = time.time()
return
duration = time.time() - start_time
progress_size = int(count * block_size)
speed = int(progress_size / (1024 * duration))
percent = int(count * block_size * 100 / total_size)
sys.stdout.write("\r...%d%%, %d MB, %d KB/s, %d seconds passed" %
(percent, progress_size / (1024 * 1024), speed, duration))
sys.stdout.flush()

# read data into numpy array

mat = scipy.io.loadmat( '/content/drive/My Drive/Hyperspectral image segmentation/Pavia
img = mat[ 'paviaU' ]
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 16/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# create a hyperspectral dataset object from the numpy array

hypData = HypImg( img )

img.shape

(610, 340, 103)

print(hypData.numRows,hypData.numCols,hypData.numBands,hypData.numClasses,hypData.numSa

610 340 103 None 207400

# pre-process data to make the model easier to train

hypData.pre_process( 'minmax' )

# create data iterator objects for training and validation using the pre-processed data
trainSamples = 200000
200
valSamples = 100
dataTrain = Iterator( dataSamples=hypData.spectraPrep[:trainSamples, :],
targets=hypData.spectraPrep[:trainSamples, :], batchSize=
dataVal = Iterator( dataSamples=hypData.spectraPrep[trainSamples:trainSamples+valSample
targets=hypData.spectraPrep[trainSamples:trainSamples+valSa

# shuffle training data

dataTrain.shuffle()

# setup a fully-connected autoencoder neural network with 3 encoder layers

net = mlp_1D_network( inputSize=hypData.numBands, encoderSize=[50,30,10], activationFu
weightInitOpt='truncated_normal', tiedWeights=No

# setup a training operation for the network

net.add_train_op( name='csa', lossFunc='CSA', learning_rate=1e-3, decay_steps=None, de
method='Adam', wd_lambda=0.0 )

# create a directory to save the learnt model

import os
model_dir = os.path.join('/content/drive/My Drive/Hyperspectral image segmentation/mode
if os.path.exists(model_dir):
# if directory already exists, delete it
shutil.rmtree(model_dir)
os.mkdir(model_dir)

# train the network for 100 epochs, saving the model at epoch 50 and 100
net.train(dataTrain=dataTrain, dataVal=dataVal, train_op_name='csa', n_epochs=100, save

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 17/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

epoch: 0, training loss: 96.8747

epoch: 0, validation loss: 0.554404
epoch: 10, training loss: 2.30624
epoch: 10, validation loss: 0.136573
epoch: 20, training loss: 1.93002
epoch: 20, validation loss: 0.118017
epoch: 30, training loss: 1.73601
epoch: 30, validation loss: 0.108916
epoch: 40, training loss: 1.65705
epoch: 40, validation loss: 0.104197
epoch: 50, training loss: 1.58422
epoch: 50, validation loss: 0.0973074
epoch: 60, training loss: 1.56249
epoch: 60, validation loss: 0.095769
epoch: 70, training loss: 1.55322
epoch: 70, validation loss: 0.0951852
epoch: 80, training loss: 1.54373
epoch: 80, validation loss: 0.0952137
epoch: 90, training loss: 1.39018
epoch: 90, validation loss: 0.0891968
epoch: 100, training loss: 1.37009
# read data into numpy array
epoch: 100, validation loss: 0.0885625
mat = scipy.io.loadmat( '/content/drive/My Drive/Hyperspectral image segmentation/Pavia
img = mat[ 'paviaU' ]

# create a hyperspectral dataset object from the numpy array

hypData = HypImg( img )

# pre-process data to make the model easier to train

hypData.pre_process( 'minmax' )

# setup a network from a config file

net = mlp_1D_network( configFile=os.path.join('/content/drive/My Drive/Hyperspectral im

# assign previously trained parameters to the network, and name model

net.add_model( addr=os.path.join('/content/drive/My Drive/Hyperspectral image segmentat

hypData.spectraPrep

array([[0.14281718, 0.10112704, 0.09126789, ..., 0.86788761, 0.87267634,

0.87605662],
[0.16696055, 0.14653082, 0.13983832, ..., 0.81437161, 0.82212082,
0.84466397],
[0.22363448, 0.27657814, 0.19610377, ..., 0.93816222, 0.95383355,
0.96061033],
...,
[0.11385162, 0.02918249, 0.04603411, ..., 0.84915783, 0.85943321,
0.86600945],
[0.10683803, 0.1743594 , 0.13333376, ..., 0.84145342, 0.83974402,
0.83162436],
[0.22781815, 0.16027218, 0.05195883, ..., 0.8381299 , 0.85051998,
0.86570783]])

tf.train.Saver
https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 18/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory
tf.train.Saver

tensorflow.python.training.saver.Saver

# feed forward hyperspectral dataset through encoder (get latent encoding)

dataZ = net.encoder( modelName='csa_100', dataSamples=hypData.spectraPrep )

INFO:tensorflow:Restoring parameters from /content/drive/My Drive/Hyperspectral i

# feed forward latent encoding through decoder (get reconstruction)

dataY = net.decoder(modelName='csa_100', dataZ=dataZ)

ing parameters from /content/drive/My Drive/Hyperspectral image segmentation/model

# reshape latent encoding to original image dimensions

imgZ = np.reshape(dataZ, (hypData.numRows, hypData.numCols, -1))

# reshape reconstructed output of decoder

imgY = np.reshape(dataY, (hypData.numRows, hypData.numCols, -1))

# reshape pre-processed input

imgX = np.reshape(hypData.spectraPrep, (hypData.numRows, hypData.numCols, -1))

# visualise latent image using 3 out of the 10 dimensions

colourImg = imgZ.copy()
colourImg = colourImg[ :,:,np.argsort(-np.std(np.std(colourImg, axis=0), axis=0))[:3]
colourImg /= np.max(np.max(colourImg, axis=0), axis=0)

.pyplot as plt
image (using 3 out of the 10 dimensions)
h.join('/content/drive/My Drive/Hyperspectral image segmentation/results', 'test_mlp_la

fig = plt.figure()
plt.plot(imgZ[576, 210, :])
plt.xlabel('latent dimension')
plt.ylabel('latent value')
plt.title('meadow spectra')
plt.savefig(os.path.join('/content/drive/My Drive/Hyperspectral image segmentation/res

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 19/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

# save plot comparing pre-processed 'meadow' spectra input with decoder reconstructio
fig = plt.figure()
ax = plt.subplot(111)
ax.plot(range(hypData.numBands),imgX[576, 210, :],label='pre-processed input')
ax.plot(range(hypData.numBands),imgY[576, 210, :],label='reconstruction')
plt.xlabel('band')
plt.ylabel('value')
plt.title('meadow spectra')
ax.legend()
plt.savefig(os.path.join('/content/drive/My Drive/Hyperspectral image segmentation/res

from PIL import Image

from matplotlib import pyplot as plt

filename = '/content/drive/My Drive/Hyperspectral image segmentation/results/test_mlp_

img = Image.open( filename + '.png' )

data = np.array( img, dtype='uint8' )

np.save( filename + '.npy', data)

# visually testing our output

img_array = np.load(filename + '.npy')
plt.imshow(img_array)

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 20/21
8/7/2020 autoencoder_mpl_basic.ipynb - Colaboratory

<matplotlib.image.AxesImage at 0x7f7a5c0aaa58>

img_array.shape

(610, 340, 4)

https://colab.research.google.com/drive/1m-6ib3lR2tgWxto-3UjgXmuIigjdHUwM#scrollTo=JG2phulTI3v-&uniqifier=8&printMode=true 21/21