Version 0.9 of new exercises. Copied from private repo

cnn/cnnConvolve.m

@@ -0,0 +1,70 @@
+function convolvedFeatures = cnnConvolve(filterDim, numFilters, images, W, b)
+%cnnConvolve Returns the convolution of the features given by W and b with
+%the given images
+%
+% Parameters:
+%  filterDim - filter (feature) dimension
+%  numFilters - number of feature maps
+%  images - large images to convolve with, matrix in the form
+%           images(r, c, image number)
+%  W, b - W, b for features from the sparse autoencoder
+%         W is of shape (filterDim,filterDim,numFilters)
+%         b is of shape (numFilters,1)
+%
+% Returns:
+%  convolvedFeatures - matrix of convolved features in the form
+%                      convolvedFeatures(imageRow, imageCol, featureNum, imageNum)
+
+numImages = size(images, 3);
+imageDim = size(images, 1);
+convDim = imageDim - filterDim + 1;
+
+convolvedFeatures = zeros(convDim, convDim, numFilters, numImages);
+
+% Instructions:
+%   Convolve every filter with every image here to produce the 
+%   (imageDim - filterDim + 1) x (imageDim - filterDim + 1) x numFeatures x numImages
+%   matrix convolvedFeatures, such that 
+%   convolvedFeatures(imageRow, imageCol, featureNum, imageNum) is the
+%   value of the convolved featureNum feature for the imageNum image over
+%   the region (imageRow, imageCol) to (imageRow + filterDim - 1, imageCol + filterDim - 1)
+%
+% Expected running times: 
+%   Convolving with 100 images should take less than 30 seconds 
+%   Convolving with 5000 images should take around 2 minutes
+%   (So to save time when testing, you should convolve with less images, as
+%   described earlier)
+
+
+for imageNum = 1:numImages
+  for filterNum = 1:numFilters
+
+    % convolution of image with feature matrix
+    convolvedImage = zeros(convDim, convDim);
+    % Obtain the feature (filterDim x filterDim) needed during the convolution
+    %%% YOUR CODE HERE %%%
+
+    % Flip the feature matrix because of the definition of convolution, as explained later
+    filter = rot90(squeeze(filter),2);
+
+    % Obtain the image
+    im = squeeze(images(:, :, imageNum));
+
+    % Convolve "filter" with "im", adding the result to convolvedImage
+    % be sure to do a 'valid' convolution
+    %%% YOUR CODE HERE %%%
+
+
+    % Add the bias unit
+    % Then, apply the sigmoid function to get the hidden activation
+    %%% YOUR CODE HERE %%%
+
+
+
+    convolvedFeatures(:, :, filterNum, imageNum) = convolvedImage;
+  end
+end
+
+
+end
+

cnn/cnnCost.m

@@ -0,0 +1,137 @@
+function [cost, grad, preds] = cnnCost(theta,images,labels,numClasses,...
+                                filterDim,numFilters,poolDim,pred)
+% Calcualte cost and gradient for a single layer convolutional neural
+% network followed by a softmax layer with cross entropy objective.
+%                            
+% Parameters:
+%  theta      -  unrolled parameter vector
+%  images     -  stores images in imageDim x imageDim x numImges array
+%  numClasses -  number of classes to predict
+%  filterDim  -  dimension of convolutional filter                            
+%  numFilters -  number of convolutional filters
+%  poolDim    -  dimension of pooling area
+%  pred       -  boolean only forward propagate and return predictions
+%
+%
+% Returns:
+%  cost       -  cross entropy cost
+%  grad       -  gradient with respect to theta (if pred==False)
+%  preds      -  list of predictions for each example (if pred==True)
+
+
+if ~exist('pred','var')
+    pred = false;
+end;
+
+
+imageDim = size(images,1); % height/width of image
+numImages = size(images,3); % number of images
+
+%% Reshape parameters and setup gradient matrices
+
+% Wc is filterDim x filterDim x numFilters parameter matrix
+% bc is the corresponding bias
+
+% Wd is numClasses x hiddenSize parameter matrix where hiddenSize is the
+% number of output units from the convolutional layer
+% bd is corresponding bias
+[Wc, Wd, bc, bd] = cnnParamsToStack(theta,imageDim,filterDim,numFilters,...
+                        poolDim,numClasses);
+
+% Same sizes as Wc,Wd,bc,bd.  Used to hold gradient w.r.t above params.
+Wc_grad = zeros(size(Wc));
+Wd_grad = zeros(size(Wd));
+bc_grad = zeros(size(bc));
+bd_grad = zeros(size(bd));
+
+
+%%======================================================================
+%% STEP 1a: Forward Propagation
+%  In this step you will forward propagate the input through the
+%  convolutional and subsampling (mean pooling) layers.  You will then use
+%  the responses from the convolution and pooling layer as the input to a
+%  standard softmax layer.
+
+%% Convolutional Layer
+%  For each image and each filter, convolve the image with the filter, add
+%  the bias and apply the sigmoid nonlinearity.  Then subsample the 
+%  convolved activations with mean pooling.  Store the results of the
+%  convolution in activations and the results of the pooling in
+%  activationsPooled.  You will need to save the convolved activations for
+%  backpropagation.
+convDim = imageDim-filterDim+1; % dimension of convolved output
+outputDim = (convDim)/poolDim; % dimension of subsampled output
+
+% convDim x convDim x numFilters x numImages tensor for storing activations
+activations = zeros(convDim,convDim,numFilters,numImages);
+
+% outputDim x outputDim x numFilters x numImages tensor for storing
+% subsampled activations
+activationsPooled = zeros(outputDim,outputDim,numFilters,numImages);
+
+%%% YOUR CODE HERE %%%
+
+
+% Reshape activations into 2-d matrix, hiddenSize x numImages,
+% for Softmax layer
+activationsPooled = reshape(activationsPooled,[],numImages);
+
+%% Softmax Layer
+%  Forward propagate the pooled activations calculated above into a
+%  standard softmax layer. For your convenience we have reshaped
+%  activationPooled into a hiddenSize x numImages matrix.  Store the
+%  results in probs.
+
+% numClasses x numImages for storing probability that each image belongs to
+% each class.
+probs = zeros(numClasses,numImages);
+
+%%% YOUR CODE HERE %%%
+
+
+%%======================================================================
+%% STEP 1b: Calculate Cost
+%  In this step you will use the labels given as input and the probs
+%  calculate above to evaluate the cross entropy objective.  Store your
+%  results in cost.
+
+cost = 0; % save objective into cost
+
+%%% YOUR CODE HERE %%%
+
+
+% Makes predictions given probs and returns without backproagating errors.
+if pred
+    [~,preds] = max(probs,[],1);
+    preds = preds';
+    grad = 0;
+    return;
+end;
+
+%%======================================================================
+%% STEP 1c: Backpropagation
+%  Backpropagate errors through the softmax and convolutional/subsampling
+%  layers.  Store the errors for the next step to calculate the gradient.
+%  Backpropagating the error w.r.t the softmax layer is as usual.  To
+%  backpropagate through the pooling layer, you will need to upsample the
+%  error with respect to the pooling layer for each filter and each image.  
+%  Use the kron function and a matrix of ones to do this upsampling 
+%  quickly.
+
+%%% YOUR CODE HERE %%%
+
+%%======================================================================
+%% STEP 1c: Gradient Calculation
+%  After backpropagating the errors above, we can use them to calculate the
+%  gradient with respect to all the parameters.  The gradient w.r.t the
+%  softmax layer is calculated as usual.  To calculate the gradient w.r.t.
+%  a filter in the convolutional layer, convolve the backpropagated error
+%  for that fileter with each image and aggregate over images.
+
+%%% YOUR CODE HERE %%%
+
+
+%% Unroll gradient into grad vector for minFunc
+grad = [Wc_grad(:) ; Wd_grad(:) ; bc_grad(:) ; bd_grad(:)];
+
+end

cnn/cnnExercise.m

@@ -0,0 +1,106 @@
+%% Convolution and Pooling Exercise
+
+%  Instructions
+%  ------------
+% 
+%  This file contains code that helps you get started on the
+%  convolution and pooling exercise. In this exercise, you will only
+%  need to modify cnnConvolve.m and cnnPool.m. You will not need to modify
+%  this file.
+
+%%======================================================================
+%% STEP 0: Initialization and Load Data
+%  Here we initialize some parameters used for the exercise.
+
+imageDim = 28;         % image dimension
+
+filterDim = 8;          % filter dimension
+numFilters = 100;         % number of feature maps
+
+numImages = 60000;    % number of images
+
+poolDim = 3;          % dimension of pooling region
+
+% Here we load MNIST training images
+addpath ../common/;
+images = loadMNISTImages('../common/train-images-idx3-ubyte');
+images = reshape(images,imageDim,imageDim,numImages);
+
+W = randn(filterDim,filterDim,numFilters);
+b = rand(numFilters);
+
+%%======================================================================
+%% STEP 1: Implement and test convolution
+%  In this step, you will implement the convolution and test it on
+%  on a small part of the data set to ensure that you have implemented
+%  this step correctly.
+
+%% STEP 1a: Implement convolution
+%  Implement convolution in the function cnnConvolve in cnnConvolve.m
+
+%% Use only the first 8 images for testing
+convImages = images(:, :, 1:8); 
+
+% NOTE: Implement cnnConvolve in cnnConvolve.m first!
+convolvedFeatures = cnnConvolve(filterDim, numFilters, convImages, W, b);
+
+%% STEP 1b: Checking your convolution
+%  To ensure that you have convolved the features correctly, we have
+%  provided some code to compare the results of your convolution with
+%  activations from the sparse autoencoder
+
+% For 1000 random points
+for i = 1:1000   
+    filterNum = randi([1, numFilters]);
+    imageNum = randi([1, 8]);
+    imageRow = randi([1, imageDim - filterDim + 1]);
+    imageCol = randi([1, imageDim - filterDim + 1]);    
+
+    patch = convImages(imageRow:imageRow + filterDim - 1, imageCol:imageCol + filterDim - 1, imageNum);
+
+    feature = sum(sum(patch.*W(:,:,filterNum)))+b(filterNum);
+    feature = 1./(1+exp(-feature));
+
+    if abs(feature - convolvedFeatures(imageRow, imageCol,filterNum, imageNum)) > 1e-9
+        fprintf('Convolved feature does not match test feature\n');
+        fprintf('Filter Number    : %d\n', filterNum);
+        fprintf('Image Number      : %d\n', imageNum);
+        fprintf('Image Row         : %d\n', imageRow);
+        fprintf('Image Column      : %d\n', imageCol);
+        fprintf('Convolved feature : %0.5f\n', convolvedFeatures(imageRow, imageCol, filterNum, imageNum));
+        fprintf('Test feature : %0.5f\n', feature);       
+        error('Convolved feature does not match test feature');
+    end 
+end
+
+disp('Congratulations! Your convolution code passed the test.');
+
+%%======================================================================
+%% STEP 2: Implement and test pooling
+%  Implement pooling in the function cnnPool in cnnPool.m
+
+%% STEP 2a: Implement pooling
+% NOTE: Implement cnnPool in cnnPool.m first!
+pooledFeatures = cnnPool(poolDim, convolvedFeatures);
+
+%% STEP 2b: Checking your pooling
+%  To ensure that you have implemented pooling, we will use your pooling
+%  function to pool over a test matrix and check the results.
+
+testMatrix = reshape(1:64, 8, 8);
+expectedMatrix = [mean(mean(testMatrix(1:4, 1:4))) mean(mean(testMatrix(1:4, 5:8))); ...
+                  mean(mean(testMatrix(5:8, 1:4))) mean(mean(testMatrix(5:8, 5:8))); ];
+
+testMatrix = reshape(testMatrix, 8, 8, 1, 1);
+
+pooledFeatures = squeeze(cnnPool(4, testMatrix));
+
+if ~isequal(pooledFeatures, expectedMatrix)
+    disp('Pooling incorrect');
+    disp('Expected');
+    disp(expectedMatrix);
+    disp('Got');
+    disp(pooledFeatures);
+else
+    disp('Congratulations! Your pooling code passed the test.');
+end

cnn/cnnInitParams.m

@@ -0,0 +1,43 @@
+function theta = cnnInitParams(imageDim,filterDim,numFilters,...
+                                poolDim,numClasses)
+% Initialize parameters for a single layer convolutional neural
+% network followed by a softmax layer.
+%                            
+% Parameters:
+%  imageDim   -  height/width of image
+%  filterDim  -  dimension of convolutional filter                            
+%  numFilters -  number of convolutional filters
+%  poolDim    -  dimension of pooling area
+%  numClasses -  number of classes to predict
+%
+%
+% Returns:
+%  theta      -  unrolled parameter vector with initialized weights
+
+%% Initialize parameters randomly based on layer sizes.
+assert(filterDim < imageDim,'filterDim must be less that imageDim');
+
+Wc = 1e-1*randn(filterDim,filterDim,numFilters);
+
+outDim = imageDim - filterDim + 1; % dimension of convolved image
+
+% assume outDim is multiple of poolDim
+assert(mod(outDim,poolDim)==0,...
+       'poolDim must divide imageDim - filterDim + 1');
+
+outDim = outDim/poolDim;
+hiddenSize = outDim^2*numFilters;
+
+r  = sqrt(6) / sqrt(numClasses+hiddenSize+1);   % we'll choose weights uniformly from the interval [-r, r]
+Wd = rand(numClasses, hiddenSize) * 2 * r - r;
+
+bc = zeros(numFilters, 1);
+bd = zeros(numClasses, 1);
+
+% Convert weights and bias gradients to the vector form.
+% This step will "unroll" (flatten and concatenate together) all 
+% your parameters into a vector, which can then be used with minFunc. 
+theta = [Wc(:) ; Wd(:) ; bc(:) ; bd(:)];
+
+end
+