\documentclass[10pt]{beamer}

% for themes, etc.
\mode<presentation>
{ \usetheme{default} }
%{ \usetheme{boxes} }

\usepackage{times}  % fonts are up to you
\usepackage{graphics}
\usepackage{amsmath}
\definecolor{red}{rgb}{0.5,0.0,0.0}
\renewcommand{\emph}[1]{\textcolor{red}{\em\bf #1}}

% these will be used later in the title page
\title{Fundamentals of Computational Neuroscience 2e}
%\author{Thomas Trappenberg}


\begin{document}

%%%%%%%%%%%% Slide %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\titlepage

\date{\today}

\begin{center} {Chapter 6: Feed-forward mapping networks} \end{center}


\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Digital representation of a letter}

\begin{center}
\resizebox{5cm}{!}{\includegraphics{figures/Chapter6/A_letter.pdf}}
\resizebox{6cm}{!}{\includegraphics{figures/Chapter6/A_binarization.pdf}}
\end{center}

\emph{Optical character recognition}: Predict meaning from features. \\
E.g., given features $\mathbf{x}$, what is the character $\mathbf{y}$
$$  f:\mathbf{x} \in \mathbf{S}_1^n \rightarrow \mathbf{y} \in \mathbf{S}_2^m$$

\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Examples given by lookup table}

Boolean AND  function

\begin{tabular}{ccc}
  \hline
  $\; x_1 \;$ & $\; x_2 \;$ & $\; y \;$ \\ \hline
  0 & 0 & 1 \\
  0 & 1 & 0 \\
  1 & 0 & 0 \\
  1 & 1 & 1 \\ \hline
\end{tabular}

\vspace*{1cm}

Look-up table for a non-boolean example function


\begin{tabular}{ccc}
  % after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
  \hline
  $x_1$ & $x_2$ & $y$ \\ \hline
  1 & 2 & -1 \\
  2 & 1 & 1 \\
  3 & -2 & 5 \\
  -1 & -1 & 7 \\ 
     ... & ... & ... \\ \hline
\end{tabular}



\end{frame} 
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large The population node as perceptron}


\emph{Update rule:} $  \mathbf{r}^{\rm out}=g(\mathbf{w} \mathbf{r}^{\rm in})  $ 
{\small (component-wise: $ r^{\rm out}_i=g(\sum_j w_{ij} r^{\rm in}_j)$)}

For example:  $r_i^{\rm in}  = x_i$, $  \tilde{y} = r^{\rm out}$, linear grain function $g(x)=x$:

$$\tilde{y} = w_1 x_1  + w_2 x_2 $$

\begin{center}
\resizebox{4cm}{!}{\includegraphics{figures/Chapter6/sigma_node2.pdf}}
\resizebox{5cm}{!}{\includegraphics{figures/Chapter6/surface1.pdf}}
\end{center}

\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large How to find the right weight values?}

\emph{Objective (error) function}, for example: mean square error (MSE)
$$ E=\frac{1}{2} \sum_i (r^{\rm out}_i - y_i)^2 $$ 

\emph{Gradient descent} method: 
$ w_{ij} \leftarrow w_{ij}  - \epsilon \frac{\partial E}{\partial w_{ij}} $ \\
$ \hspace*{4.9cm} = w_{ij} - \epsilon (y_i-r^{\rm out}_i) r^{\rm in}_j $  \hspace*{0.5cm} {\tiny for MSE, linear gain}
\begin{center}
\resizebox{4cm}{!}{\includegraphics{figures/Chapter6/gradient_decent.pdf}}
{\small
\begin{tabular}{|l|}
\hline
 Initialize weights arbitrarily  \\
 Repeat until error is sufficiently small \\
 \hspace*{0.6cm}  Apply a sample pattern to the input nodes:  $ r^0_i = r^{\rm in}_i = \xi^{\rm in}_i $ \\
 \hspace*{0.6cm}  Calculate rate of the output nodes: $ r^{\rm out}_i = g(\sum_j w_{ij} r_j^{\rm in})$ \\
 \hspace*{0.6cm} Compute the delta term for the output layer: $ \delta_i = g'(h^{\rm out}_i) (\xi^{\rm out}_i - r^{\rm out}_i)$ \\
 \hspace*{0.6cm} Update the weight matrix by adding the term: $ \Delta w_{ij} = \epsilon \delta_i r^{\rm in}_j$ \\
 \hline
\end{tabular}
}
\end{center}

\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Example: OCR}

\begin{center}
\resizebox{10cm}{!}{\includegraphics{figures/Chapter6/perceptronExample.pdf}}
\end{center}

\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Example: Boolean function}

\begin{center}
\resizebox{8cm}{!}{\includegraphics{figures/Chapter6/boolean.pdf}}
\end{center}


\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\frame[containsverbatim] { \frametitle{\large perceptronTrain.m}
\scriptsize

\begin{verbatim}
 1   %% Letter recognition with threshold perceptron
 2    clear; clf;
 3    nIn=12*13; nOut=26;
 4    wOut=rand(nOut,nIn)-0.5;
 5
 6   % training vectors
 7    load pattern1;
 8    rIn=reshape(pattern1', nIn, 26);
 9    rDes=diag(ones(1,26));
10
11   % Updating and training network
12    for training_step=1:20;
13        % test all pattern
14         rOut=(wOut*rIn)>0.5;
15         distH=sum(sum((rDes-rOut).^2))/26;
16         error(training_step)=distH;
17        % training with delta rule
18         wOut=wOut+0.1*(rDes-rOut)*rIn';
19    end
20
21    plot(0:19,error)
22    xlabel('Training step')
23    ylabel('Average Hamming distance')
\end{verbatim}
}

%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large The mulitlayer Perceptron (MLP)}

\begin{center}
\resizebox{5cm}{!}{\includegraphics{figures/Chapter6/mlp.pdf}}
\end{center}

Update rule:
$  \mathbf{r}^{\rm out}=g^{\rm out} (\mathbf{w}^{\rm out} g^{\rm h}(\mathbf{w}^{\rm h} \mathbf{r}^{\rm
  in})) $

\vspace*{0.5cm}
Learning rule (error backpropagation): $ w_{ij} \leftarrow w_{ij}  - \epsilon \frac{\partial E}{\partial w_{ij}} $ 


\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large The error-backpropagation algorithm}

{\small

\begin{tabular}{| l |}
  % after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
  \hline
  Initialize weights arbitrarily  \\
  Repeat until error is sufficiently small \\
  \hspace*{0.6cm} Apply a sample pattern to the input nodes: $ r^0_i := r^{\rm in}_i = \xi^{\rm in}_i $ \\
  \hspace*{0.6cm} Propagate input through the network by calculating the rates of nodes in \\
  \hspace*{0.6cm} successive layers $l$:  $ r^{l}_i = g(h^{l}_i)= g(\sum_j w^{l}_{ij} r_j^{l-1})$ \\
  \hspace*{0.6cm} Compute the delta term for the output layer: $ \delta^{\rm out}_i = g'(h^{\rm out}_i) (\xi^{\rm out}_i - r^{\rm out}_i)$ \\
  \hspace*{0.6cm} Back-propagate delta terms through the network: $ \delta^{l-1}_i = g'(h^{l-1}_i) \sum_j w^{l}_{ji} \delta^l_j $ \\
  \hspace*{0.6cm} Update weight matrix by adding the term: $ \Delta w_{ij}^l = \epsilon \delta^l_i r^{l-1}_j$ \\
  \hline
\end{tabular}
}


\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\frame[containsverbatim] { \frametitle{\large mlp.m}
\scriptsize

\begin{verbatim}
 1   %% MLP with backpropagation learning on XOR problem
 2    clear; clf;
 3    N_i=2; N_h=2; N_o=1;
 4    w_h=rand(N_h,N_i)-0.5; w_o=rand(N_o,N_h)-0.5;
 5
 6    % training vectors (XOR)
 7    r_i=[0 1 0 1 ; 0 0 1 1];
 8    r_d=[0 1 1 0];
 9
10    % Updating and training network with sigmoid activation function
11    for sweep=1:10000;
12      % training randomly on one pattern
13        i=ceil(4*rand);
14        r_h=1./(1+exp(-w_h*r_i(:,i)));
15        r_o=1./(1+exp(-w_o*r_h));
16        d_o=(r_o.*(1-r_o)).*(r_d(:,i)-r_o);
17        d_h=(r_h.*(1-r_h)).*(w_o'*d_o);
18        w_o=w_o+0.7*(r_h*d_o')';
19        w_h=w_h+0.7*(r_i(:,i)*d_h')';
20      % test all pattern
21        r_o_test=1./(1+exp(-w_o*(1./(1+exp(-w_h*r_i)))));
22        d(sweep)=0.5*sum((r_o_test-r_d).^2);
23    end
24    plot(d)
\end{verbatim}
}

%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large MLP for XOR function}
\begin{center}
\resizebox{5cm}{!}{\includegraphics{figures/Chapter6/mlp_XOR.pdf}}
\hspace*{1cm}
\resizebox{4.5cm}{!}{\includegraphics{figures/Chapter6/mlp_XOR2.pdf}}
\end{center}

\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large MLP approximating sine function}
\begin{center}
\resizebox{7cm}{!}{\includegraphics{figures/mlp_sine_slides.pdf}}
\end{center}


\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Overfitting and underfitting}

\begin{center}
\resizebox{5cm}{!}{\includegraphics{figures/Chapter6/overfit.pdf}}
\end{center}

Regularization, for example 
$$ E= \frac{1}{2} \sum_i (r^{\rm out}_i -y_i)^2 - \gamma_r \frac{1}{2} \sum_i w_i^2 $$




\end{frame}
%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Support Vector Machines}

\begin{center}
\resizebox{4.5cm}{!}{\includegraphics{figures/Chapter6/svm1.pdf}}
\hspace*{2cm}\resizebox{3.5cm}{!}{\includegraphics{figures/Chapter6/svm1nl.pdf}}
\end{center}


\end{frame}

%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large SVM: Kernel trick}

\begin{center}
\resizebox{10cm}{!}{\includegraphics{figures/Chapter6/svm1map.pdf}}
\end{center}


\end{frame}


%%%%%%%%%%%% Slide %%%%%%%%%%%
\begin{frame}
\frametitle{\large Further Readings} 

\begin{list}{}{\setlength\itemindent{-.3in}  \setlength\rightmargin{.1in} \setlength\itemsep{0.05in}}
\footnotesize

\item Simon Haykin (1999), \emph{Neural networks: a comprehensive foundation}, MacMillan (2nd edition). 

\item John Hertz, Anders Krogh, and Richard G.~Palmer (1991), \emph{Introduction
to the theory of neural computation}, Addison-Wesley.

\item Berndt M\"uller, Joachim Reinhardt, and Michael Thomas Strickland (1995), \emph{Neural Networks: An Introduction}, Springer

\item Christopher M. Bishop (2006), \emph{Pattern Recognition and Machine Learning}, Springer

\item Laurence F.~Abbott and Sacha B.~Nelson (2000), \emph{Synaptic
plasticity: taming the beast}, in \emph{Nature Neurosci.~(suppl.)}, 3: 1178--83.

\item Christopher J.~C.~Burges (1998), \emph{A Tutorial on Support Vector Machines for Pattern Recognition} in \emph{Data Mining and Knowledge Discovery} 2:121--167.

\item Alex J.~Smola and Bernhard Sch\"olhopf (2004), \emph{A tutorial on support vector regression} in \emph{Statistics and computing} 14: 199-222.

\item David E. Rumelhart, James L. McClelland, and the PDP research group (1986),
 \emph{Parallel Distributed Processing: Explorations in the Microstructure of Cognition}, MIT Press. 
 
\item Peter McLeod, Kim Plunkett, and Edmund T.~Rolls (1998), \emph{Introduction
to connectionist modelling of cognitive processes}, Oxford University Press.

\item  E.~Bruce Goldstein (1999), \emph{Sensation \& perception}, Brooks/Cole Publishing Company (5th edition).

\end{list} 

\end{frame}

\end{document}







