%% MLP with backpropagation learning of AND function
%% TT 2/2/16
clear; close all; hold on;

% training vectors (AND)
X=[0 0 1 1; ...
   0 1 0 1; ...
   1 1 1 1];

Y=[0 0 0 1];

% model specifications
Ni=size(X,1); Nh=3; No=size(Y,1);
wh=randn(Nh,Ni); wo=randn(No,Nh); dwo=zeros(size(wo)); dwh=zeros(size(wh));

% Applying and training network with sigmoid activation function
for trial=1:40000;
    %pick example pattern randomly
    pat=randi(4); x=X(:,pat);
    for ih=1:Nh;  %for each hidden node
        sumInput=0;
        for in=1:Ni  %loop over input features
            sumInput=sumInput+wh(ih,in)*x(in);
        end
        h(ih)=1/(1+exp(-sumInput));
    end
    for io=1:No;  %for each output node
        sumInput=0;
        for ih=1:Nh  %loop over inputs from hidden
            sumInput=sumInput+wo(io,ih)*h(ih);
        end
        y(io)=1/(1+exp(-sumInput));
    end
    
    % delta term for each layer (objective function error)
    
    for io=1:No;  %for each output node
        do(io)=y(io)*(1-y(io))*(Y(pat)-y(io));
    end
    
    sumInput=0;
    for ih=1:Nh;  %for each hidden node
        sumInput=0;   %backpropgation starts
        for io=1:No
            sumInput=sumInput+wo(io,ih)*do(io);
        end
        dh(ih)=h(ih)*(1-h(ih))*sumInput;
    end
    
    %update weights with momentum
    for io=1:No;
        for ih=1:Nh;
            dwo(io,ih)=0.9*dwo(io,ih)+h(ih)*do(io);
            wo(io,ih)=wo(io,ih)+0.1*dwo(io,ih);
        end
    end
    for ih=1:Nh;
        for in=1:Ni
            dwh(ih,in)=0.9*dwh(ih,in)+x(in)*dh(ih);
            wh(ih,in)=wh(ih,in)+0.1*dwh(ih,in);
        end
    end
    % test error
    d(trial)=abs(y-Y(pat));
    
end
plot(d); 