function [net, info] = dldl_pose_train(net, imdb, getBatch, varargin)
%CNN_TRAIN  An example implementation of SGD for training CNNs
%    CNN_TRAIN() is an example learner implementing stochastic
%    gradient descent with momentum to train a CNN. It can be used
%    with different datasets and tasks by providing a suitable
%    getBatch function.
%
%    The function automatically restarts after each training epoch by
%    checkpointing.
%
%    The function supports training on CPU or on one or more GPUs
%    (specify the list of GPU IDs in the `gpus` option). Multi-GPU
%    support is relatively primitive but sufficient to obtain a
%    noticable speedup.

% Copyright (C) 2014-15 Andrea Vedaldi.
% All rights reserved.
%
% This file is part of the VLFeat library and is made available under
% the terms of the BSD license (see the COPYING file).

opts.batchSize = 256 ;
opts.numSubBatches = 1 ;
opts.train = [] ;
opts.val = [] ;
opts.numEpochs = 300 ;
opts.gpus = [] ; % which GPU devices to use (none, one, or more)
opts.learningRate = 0.001 ;
opts.continue = false ;
opts.expDir = fullfile('data','exp') ;
opts.conserveMemory = false ;
opts.backPropDepth = +inf ;
opts.sync = false ;
opts.prefetch = false ;
opts.cudnn = true ;
opts.weightDecay = 0.0005 ;
opts.momentum = 0.9 ;
opts.loss = 'softmaxloss';
% opts.errorFunction = 'multiclass' ;
opts.errorFunction = 'regression' ;

opts.errorLabels = {} ;
opts.plotDiagnostics = false ;
opts.memoryMapFile = fullfile(tempdir, 'matconvnet.bin') ;

opts.data_type = 'pose';
opts.dataset = 'point04';

opts = vl_argparse(opts, varargin) ;
switch opts.loss
    case 'softmaxloss'
        opts.errorFunction = 'multiclass' ;
    case {'l2loss','l1loss','epsilonloss','klgloss','tukeyloss'}
        opts.errorFunction = 'singleregression' ;
    otherwise 
        opts.errorFunction = 'regression' ;
end
if ~exist(opts.expDir, 'dir'), mkdir(opts.expDir) ; end
if isempty(opts.train), opts.train = find(imdb.images.set==1) ; end
if isempty(opts.val), opts.val = find(imdb.images.set==2) ; end
if isnan(opts.train), opts.train = [] ; end

% -------------------------------------------------------------------------
%                                                    Network initialization
% -------------------------------------------------------------------------

evaluateMode = isempty(opts.train) ;

if ~evaluateMode
  for i=1:numel(net.layers)
    if isfield(net.layers{i}, 'weights')
      J = numel(net.layers{i}.weights) ;
      for j=1:J
        net.layers{i}.momentum{j} = zeros(size(net.layers{i}.weights{j}), 'single') ;
      end
      if ~isfield(net.layers{i}, 'learningRate')
        net.layers{i}.learningRate = ones(1, J, 'single') ;
      end
      if ~isfield(net.layers{i}, 'weightDecay')
        net.layers{i}.weightDecay = ones(1, J, 'single') ;
      end
    end
    % Legacy code: will be removed
    if isfield(net.layers{i}, 'filters')
      net.layers{i}.momentum{1} = zeros(size(net.layers{i}.filters), 'single') ;
      net.layers{i}.momentum{2} = zeros(size(net.layers{i}.biases), 'single') ;
      if ~isfield(net.layers{i}, 'learningRate')
        net.layers{i}.learningRate = ones(1, 2, 'single') ;
      end
      if ~isfield(net.layers{i}, 'weightDecay')
        net.layers{i}.weightDecay = single([1 0]) ;
      end
    end
  end
end

% setup GPUs
numGpus = numel(opts.gpus) ;
if numGpus > 1
  if isempty(gcp('nocreate')),
    parpool('local',numGpus) ;
    spmd, gpuDevice(opts.gpus(labindex)), end
  end
elseif numGpus == 1
  gpuDevice(opts.gpus)
end
if exist(opts.memoryMapFile), delete(opts.memoryMapFile) ; end

% setup error calculation function
if isstr(opts.errorFunction)
  switch opts.errorFunction
    case 'none'
      opts.errorFunction = @error_none ;
    case 'multiclass'
      opts.errorFunction = @error_multiclass ;
     if isempty(opts.errorLabels), opts.errorLabels = {'MaePitch','MaeYaw','MaePitchYaw','AccPitch','AccYaw','PitchYaw'} ; end
      case 'regression'
      opts.errorFunction = @error_regression ;
      switch opts.dataset
          case 'aflw_det'
              if isempty(opts.errorLabels), opts.errorLabels = {'MaeYaw','MaePitch','MaeYP','AccYaw','AccPitch','AccYP'} ; end
          otherwise
              if isempty(opts.errorLabels), opts.errorLabels = {'MaePitch','MaeYaw','MaePitchYaw','AccPitch','AccYaw','PitchYaw'} ; end
      end
      if isempty(opts.errorLabels), opts.errorLabels = {'MaePitch','MaeYaw','MaePitchYaw','AccPitch','AccYaw','PitchYaw'} ; end
      case 'singleregression'
          opts.errorFunction = @error_singleregression ;
          switch opts.dataset
              case 'aflw_det'
                  if isempty(opts.errorLabels), opts.errorLabels = {'MaeYaw','MaePitch','MaeYP','AccYaw','AccPitch','AccYP'} ; end
              otherwise
                  if isempty(opts.errorLabels), opts.errorLabels = {'MaePitch','MaeYaw','MaePitchYaw','AccPitch','AccYaw','PitchYaw'} ; end
          end
      case 'binary'
      opts.errorFunction = @error_binary ;
      if isempty(opts.errorLabels), opts.errorLabels = {'bine'} ; end
    otherwise
      error('Uknown error function ''%s''', opts.errorFunction) ;
  end
end

% -------------------------------------------------------------------------
%                                                        Train and validate
% -------------------------------------------------------------------------

modelPath = @(ep) fullfile(opts.expDir, sprintf('net-epoch-%d.mat', ep));
modelFigPath = fullfile(opts.expDir, 'net-train.pdf') ;

 start = opts.continue * findLastCheckpoint(opts.expDir) ;
% start = 1 ;

if start >= 1
  fprintf('resuming by loading epoch %d\n', start) ;
  load(modelPath(start), 'net', 'info') ;
end

for epoch=start+1:opts.numEpochs

  % train one epoch and validate
  learningRate = opts.learningRate(min(epoch, numel(opts.learningRate))) ;
  train = opts.train(randperm(numel(opts.train))) ; % shuffle
  val = opts.val ;
  if numGpus <= 1
      [net,stats.train] = process_epoch(opts, getBatch, epoch, train, learningRate, imdb, net) ;
      %       [~,stats.train] = process_epoch(opts, getBatch, epoch, train, 1, imdb, net) ;
      [~,stats.val] = process_epoch(opts, getBatch, epoch, val, 0, imdb, net) ;
  else
      spmd(numGpus)
          [net_, stats_train_] = process_epoch(opts, getBatch, epoch, train, learningRate, imdb, net) ;
          [~, stats_val_] = process_epoch(opts, getBatch, epoch, val, 0, imdb, net_) ;
      end
      net = net_{1} ;
      stats.train = sum([stats_train_{:}],2) ;
      stats.val = sum([stats_val_{:}],2) ;
  end

  % save
  if evaluateMode, sets = {'val'} ; else sets = {'train', 'val'} ; end
  for f = sets
    f = char(f) ;
    n = numel(eval(f)) ;
    info.(f).speed(epoch) = n / stats.(f)(1) * max(1, numGpus) ;
    info.(f).objective(epoch) = stats.(f)(2) / n ;
    info.(f).error(:,epoch) = stats.(f)(3:end) / n ;
  end
   if ~evaluateMode, save(modelPath(epoch), 'net', 'info') ; end

  figure(1) ; clf ;
  hasError = isa(opts.errorFunction, 'function_handle') ;
  subplot(1,1+hasError,1) ;
  if ~evaluateMode
    semilogy(1:epoch, info.train.objective, '.-', 'linewidth', 2) ;
    hold on ;
  end
  semilogy(1:epoch, info.val.objective, '.--', 'linewidth', 2) ;
  xlabel('training epoch') ; ylabel('energy') ;
  grid on ;
  h=legend(sets) ;
  set(h,'color','none');
  title('objective') ;
  if hasError
    subplot(1,2,2) ; leg = {} ;
    if ~evaluateMode
      plot(1:epoch, info.train.error', '.-', 'linewidth', 2) ;
      hold on ;
      leg = horzcat(leg, strcat('train ', opts.errorLabels)) ;
    end
    plot(1:epoch, info.val.error', '.--', 'linewidth', 2) ;
    leg = horzcat(leg, strcat('val ', opts.errorLabels)) ;
    set(legend(leg{:}),'color','none') ;
    grid on ;
    xlabel('training epoch') ; ylabel('error') ;
    title('error') ;
  end
  drawnow ;
  print(1, modelFigPath, '-dpdf') ;
end

% -------------------------------------------------------------------------
function err = error_multiclass(opts, labels, res)
% -------------------------------------------------------------------------
predictions = gather(res(end-1).x) ;
[~,predictions] = sort(predictions, 3, 'descend') ;

% be resilient to badly formatted labels
if numel(labels) == size(predictions, 4)
  labels = reshape(labels,1,1,1,[]) ;
end

% skip null labels
mass = single(labels(:,:,1,:) > 0) ;
if size(labels,3) == 2
  % if there is a second channel in labels, used it as weights
  mass = mass .* labels(:,:,2,:) ;
  labels(:,:,2,:) = [] ;
end
pre = squeeze(predictions);
mu = pre(1,:);
switch opts.dataset
    case  'bjut-3d'
        cordstruct = load('cordin.mat');
        cordin = cordstruct.cordin;
        
        cordin(:,1) = cordin(:,1)*10;
        cordin(:,2) = cordin(:,2)*10;
        
        
        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        dif_pose = abs(pre_pose-rel_pose)';
        
    case 'point04'
        cordstruct = load('cordin_angle.mat');
        cordin = cordstruct.cordin;
      
        cordin(:,1) = cordin(:,1)*15;
        cordin(:,2) = cordin(:,2)*15;
  
        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        
        dif_pose = abs(pre_pose-rel_pose)';       
    case 'aflw_det'
        cordin_yaw = repmat(-90:3:90,61,1);
        cordin_pitch = repmat([90:-3:-90]',1,61);
        cordin = [reshape(cordin_yaw,[],1),reshape(cordin_pitch,[],1)];

        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        dif_pose = abs(pre_pose-rel_pose)';  
end
err(1:2,:) = sum(dif_pose,2);
err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));
switch opts.dataset
    case 'aflw_det'
        err(4,1) = sum(dif_pose(1,:) <=15)*100;
        err(5,1) = sum(dif_pose(2,:) <=15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
    otherwise
        m_acc = abs(dif_pose) ==0;
        err(4:5,1) = sum(m_acc,2);
        err(6,1) = sum(sum(m_acc')==2);
end



function err = error_singleregression(opts, labels, res)
% -------------------------------------------------------------------------
 predictions = squeeze(gather(res(end-1).x)) ;

switch opts.dataset
    case  'bjut-3d'
        [~,ps_yaw] = mapminmax(([-60:10:60]),-1,1);
        [~,ps_pitch] = mapminmax(([-40:10:40]),-1,1);      
    case 'point04'
        VerticalAngle = [-90, -60, -30, -15, 0, +15, +30, +60, +90];
        HorizontalAngle = [-90, -75, -60, -45, -30, -15, 0, +15, +30, +45, +60, +75, +90];
        [~,ps_pitch] = mapminmax(VerticalAngle,-1,1); 
        [~,ps_yaw] = mapminmax(HorizontalAngle,-1,1);
    case 'aflw_det'
        [~,ps_yaw] = mapminmax(([-90:90]),-1,1);
        [~,ps_pitch] = mapminmax(([-90:90]),-1,1); 
%         [~,ps_roll] = mapminmax(([-90:90]),-1,1); 
end
switch  opts.dataset
    case {'bjut-3d'}
        rel_pose = labels;
        pre_pose(1,:) =  mapminmax('reverse', predictions(1,:),ps_pitch);
        pre_pose(2,:) =  mapminmax('reverse', predictions(2,:),ps_yaw);
        dif_pose = abs(pre_pose-rel_pose);
        
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <10)*100;
        err(5,1) = sum(dif_pose(2,:) <10)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=10)+double(dif_pose(2,:)<=10) ==2)*100;

    case 'point04'
        rel_pose = labels;
        pre_pose(1,:) =  mapminmax('reverse', predictions(1,:),ps_pitch);
        pre_pose(2,:) =  mapminmax('reverse', predictions(2,:),ps_yaw);
        dif_pose = abs(pre_pose-rel_pose);
        
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <15)*100;
        err(5,1) = sum(dif_pose(2,:) <15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
    case  'aflw_det' 
        rel_pose = labels;
        
        pre_pose(1,:) =  mapminmax('reverse', predictions(1,:),ps_yaw);
        pre_pose(2,:) =  mapminmax('reverse', predictions(2,:),ps_pitch);
        
        dif_pose = abs(pre_pose-rel_pose);
        % mae
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <15)*100;
        err(5,1) = sum(dif_pose(2,:) <15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
end



function err = error_regression(opts, labels, res)
% -------------------------------------------------------------------------
 predictions = squeeze(gather(res(end-1).x)) ;
 [~,mu]  = max(predictions);
switch opts.dataset
    case  'bjut-3d'
        cordstruct = load('cordin.mat');
        cordin = cordstruct.cordin;
        
        cordin(:,1) = cordin(:,1)*10;
        cordin(:,2) = cordin(:,2)*10;
        
        
        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        dif_pose = abs(pre_pose-rel_pose)';
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <15)*100;
        err(5,1) = sum(dif_pose(2,:) <15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
    case 'point04'
        cordstruct = load('cordin_angle.mat');
        cordin = cordstruct.cordin;
      
        cordin(:,1) = cordin(:,1)*15;
        cordin(:,2) = cordin(:,2)*15;
  
        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        
        dif_pose = abs(pre_pose-rel_pose)';    
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <15)*100;
        err(5,1) = sum(dif_pose(2,:) <15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
    case 'aflw_det'
        cordin_yaw = repmat(-90:3:90,61,1);
        cordin_pitch = repmat([90:-3:-90]',1,61);
        cordin = [reshape(cordin_yaw,[],1),reshape(cordin_pitch,[],1)];

        pre_pose = cordin(mu',:);
        rel_pose =  labels';
        dif_pose = abs(pre_pose-rel_pose)';  
        err(1:2,:) = sum(dif_pose,2);
        err(3,1) = sum(sqrt(dif_pose(1,:).^2+dif_pose(2,:).^2));

        err(4,1) = sum(dif_pose(1,:) <15)*100;
        err(5,1) = sum(dif_pose(2,:) <15)*100;
        err(6,1) = sum(double(dif_pose(1,:)<=15)+double(dif_pose(2,:)<=15) ==2)*100;
end





% -------------------------------------------------------------------------
function err = error_binaryclass(opts, labels, res)
% -------------------------------------------------------------------------
predictions = gather(res(end-1).x) ;
error = bsxfun(@times, predictions, labels) < 0 ;
err = sum(error(:)) ;

% -------------------------------------------------------------------------
function err = error_none(opts, labels, res)
% -------------------------------------------------------------------------
err = zeros(0,1) ;

% -------------------------------------------------------------------------
function  [net_cpu,stats,prof] = process_epoch(opts, getBatch, epoch, subset, learningRate, imdb, net_cpu)
% -------------------------------------------------------------------------

% move CNN to GPU as needed
numGpus = numel(opts.gpus) ;
if numGpus >= 1
  net = vl_simplenn_move(net_cpu, 'gpu') ;
else
  net = net_cpu ;
  net_cpu = [] ;
end

% validation mode if learning rate is zero
training = learningRate > 0 ;
if training, mode = 'training' ; else, mode = 'validation' ; end
if nargout > 2, mpiprofile on ; end

numGpus = numel(opts.gpus) ;
if numGpus >= 1
  one = gpuArray(single(1)) ;
else
  one = single(1) ;
end
res = [] ;
mmap = [] ;
stats = [] ;
start = tic ;

for t=1:opts.batchSize:numel(subset)
  fprintf('%s: epoch %02d: batch %3d/%3d: ', mode, epoch, ...
          fix(t/opts.batchSize)+1, ceil(numel(subset)/opts.batchSize)) ;
  batchSize = min(opts.batchSize, numel(subset) - t + 1) ;
  numDone = 0 ;
  error = [] ;
  for s=1:opts.numSubBatches
    % get this image batch and prefetch the next
    batchStart = t + (labindex-1) + (s-1) * numlabs ;
    batchEnd = min(t+opts.batchSize-1, numel(subset)) ;
    batch = subset(batchStart : opts.numSubBatches * numlabs : batchEnd) ;
    [im, labels] = getBatch(imdb, batch) ;
    
    if opts.prefetch
      if s==opts.numSubBatches
        batchStart = t + (labindex-1) + opts.batchSize ;
        batchEnd = min(t+2*opts.batchSize-1, numel(subset)) ;
      else
        batchStart = batchStart + numlabs ;
      end
      nextBatch = subset(batchStart : opts.numSubBatches * numlabs : batchEnd) ;
      getBatch(imdb, nextBatch) ;
    end

    if numGpus >= 1
      im = gpuArray(im) ;
    end

    % evaluate CNN
    net.layers{end}.class = labels.ld ;
    if training, dzdy = one; else, dzdy = [] ; end
    %     dzdy = [];
    res = agepose_simplenn(net, im, dzdy, res, ...
                      'accumulate', s ~= 1, ...
                      'disableDropout', ~training, ...
                      'conserveMemory', opts.conserveMemory, ...
                      'backPropDepth', opts.backPropDepth, ...
                      'sync', opts.sync, ...
                      'cudnn', opts.cudnn) ;

    % accumulate training errors
    error = sum([error, [...
      sum(double(gather(res(end).x))) ;
      reshape(opts.errorFunction(opts, labels.gt, res),[],1) ; ]],2) ;
    numDone = numDone + numel(batch) ;
  end

  % gather and accumulate gradients across labs
  if training
    if numGpus <= 1
      [net,res] = accumulate_gradients(opts, learningRate, batchSize, net, res) ;
    else
      if isempty(mmap)
        mmap = map_gradients(opts.memoryMapFile, net, res, numGpus) ;
      end
      write_gradients(mmap, net, res) ;
      labBarrier() ;
      [net,res] = accumulate_gradients(opts, learningRate, batchSize, net, res, mmap) ;
    end
  end

  % print learning statistics

  time = toc(start) ;
  stats = sum([stats,[0 ; error]],2); % works even when stats=[]
  stats(1) = time ;
  n = (t + batchSize - 1) / max(1,numlabs) ;
  speed = n/time ;
  fprintf('%.1f Hz%s\n', speed) ;

  fprintf(' obj:%.3g', stats(2)/n) ;
  for i=1:numel(opts.errorLabels)
    fprintf(' %s:%.3g', opts.errorLabels{i}, stats(i+2)/n) ;
  end
  fprintf(' [%d/%d]', numDone, batchSize);
  fprintf('\n') ;

  % debug info
  if opts.plotDiagnostics && numGpus <= 1
    figure(2) ; vl_simplenn_diagnose(net,res) ; drawnow ;
  end
end

if nargout > 2
  prof = mpiprofile('info');
  mpiprofile off ;
end

if numGpus >= 1
  net_cpu = vl_simplenn_move(net, 'cpu') ;
else
  net_cpu = net ;
end

% -------------------------------------------------------------------------
function [net,res] = accumulate_gradients(opts, lr, batchSize, net, res, mmap)
% -------------------------------------------------------------------------
for l=numel(net.layers):-1:1
  for j=1:numel(res(l).dzdw)
    thisDecay = opts.weightDecay * net.layers{l}.weightDecay(j) ;
    thisLR = lr * net.layers{l}.learningRate(j) ;

    % accumualte from multiple labs (GPUs) if needed
    if nargin >= 6
      tag = sprintf('l%d_%d',l,j) ;
      tmp = zeros(size(mmap.Data(labindex).(tag)), 'single') ;
      for g = setdiff(1:numel(mmap.Data), labindex)
        tmp = tmp + mmap.Data(g).(tag) ;
      end
      res(l).dzdw{j} = res(l).dzdw{j} + tmp ;
    end

    if isfield(net.layers{l}, 'weights')
      net.layers{l}.momentum{j} = ...
        opts.momentum * net.layers{l}.momentum{j} ...
        - thisDecay * net.layers{l}.weights{j} ...
        - (1 / batchSize) * res(l).dzdw{j} ;
      net.layers{l}.weights{j} = net.layers{l}.weights{j} + thisLR * net.layers{l}.momentum{j} ;
    else
      % Legacy code: to be removed
      if j == 1
        net.layers{l}.momentum{j} = ...
          opts.momentum * net.layers{l}.momentum{j} ...
          - thisDecay * net.layers{l}.filters ...
          - (1 / batchSize) * res(l).dzdw{j} ;
        net.layers{l}.filters = net.layers{l}.filters + thisLR * net.layers{l}.momentum{j} ;
      else
        net.layers{l}.momentum{j} = ...
          opts.momentum * net.layers{l}.momentum{j} ...
          - thisDecay * net.layers{l}.biases ...
          - (1 / batchSize) * res(l).dzdw{j} ;
        net.layers{l}.biases = net.layers{l}.biases + thisLR * net.layers{l}.momentum{j} ;
      end
    end
  end
end

% -------------------------------------------------------------------------
function mmap = map_gradients(fname, net, res, numGpus)
% -------------------------------------------------------------------------
format = {} ;
for i=1:numel(net.layers)
  for j=1:numel(res(i).dzdw)
    format(end+1,1:3) = {'single', size(res(i).dzdw{j}), sprintf('l%d_%d',i,j)} ;
  end
end
format(end+1,1:3) = {'double', [3 1], 'errors'} ;
if ~exist(fname) && (labindex == 1)
  f = fopen(fname,'wb') ;
  for g=1:numGpus
    for i=1:size(format,1)
      fwrite(f,zeros(format{i,2},format{i,1}),format{i,1}) ;
    end
  end
  fclose(f) ;
end
labBarrier() ;
mmap = memmapfile(fname, 'Format', format, 'Repeat', numGpus, 'Writable', true) ;

% -------------------------------------------------------------------------
function write_gradients(mmap, net, res)
% -------------------------------------------------------------------------
for i=1:numel(net.layers)
  for j=1:numel(res(i).dzdw)
    mmap.Data(labindex).(sprintf('l%d_%d',i,j)) = gather(res(i).dzdw{j}) ;
  end
end

% -------------------------------------------------------------------------
function epoch = findLastCheckpoint(modelDir)
% -------------------------------------------------------------------------
list = dir(fullfile(modelDir, 'net-epoch-*.mat')) ;
tokens = regexp({list.name}, 'net-epoch-([\d]+).mat', 'tokens') ;
epoch = cellfun(@(x) sscanf(x{1}{1}, '%d'), tokens) ;
epoch = max([epoch 0]) ;
