forked from BVLC/caffe
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Parallelize batches among GPUs and tree-reduce the gradients - The effective batch size scales with the number of devices - Batch size is multiplied by the number of devices - Split batches between GPUs, and tree-reduce the gradients - Detect machine topology (twin-GPU boards, P2P connectivity) - Track device in syncedmem (thanks @thatguymike) - Insert a callback in the solver for minimal code change - Accept list for gpu flag of caffe tool, e.g. '-gpu 0,1' or '-gpu all'. Run on default GPU if no ID given. - Add multi-GPU solver test - Deterministic architecture for reproducible runs
Showing
18 changed files
with
949 additions
and
141 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
#ifndef CAFFE_PARALLEL_HPP_ | ||
#define CAFFE_PARALLEL_HPP_ | ||
|
||
#include <boost/date_time/posix_time/posix_time.hpp> | ||
|
||
#include <vector> | ||
|
||
#include "caffe/blob.hpp" | ||
#include "caffe/common.hpp" | ||
#include "caffe/internal_thread.hpp" | ||
#include "caffe/layer.hpp" | ||
#include "caffe/proto/caffe.pb.h" | ||
#include "caffe/solver.hpp" | ||
#include "caffe/syncedmem.hpp" | ||
#include "caffe/util/blocking_queue.hpp" | ||
|
||
namespace caffe { | ||
|
||
// Represents a net parameters. Once a net is created, its parameter buffers can | ||
// be replaced by ones from Params, to allow parallelization. Params ensures | ||
// parameters are allocated in one consecutive array. | ||
template<typename Dtype> | ||
class Params { | ||
public: | ||
explicit Params(shared_ptr<Solver<Dtype> > root_solver); | ||
virtual ~Params() { | ||
} | ||
|
||
inline size_t size() const { | ||
return size_; | ||
} | ||
inline Dtype* data() const { | ||
return data_; | ||
} | ||
inline Dtype* diff() const { | ||
return diff_; | ||
} | ||
|
||
protected: | ||
const size_t size_; // Size of buffers | ||
Dtype* data_; // Network parameters | ||
Dtype* diff_; // Gradient | ||
|
||
DISABLE_COPY_AND_ASSIGN(Params); | ||
}; | ||
|
||
// Params stored in GPU memory. | ||
template<typename Dtype> | ||
class GPUParams : public Params<Dtype> { | ||
public: | ||
GPUParams(shared_ptr<Solver<Dtype> > root_solver, int device); | ||
virtual ~GPUParams(); | ||
|
||
void configure(Solver<Dtype>* solver) const; | ||
|
||
protected: | ||
using Params<Dtype>::size_; | ||
using Params<Dtype>::data_; | ||
using Params<Dtype>::diff_; | ||
}; | ||
|
||
class DevicePair { | ||
public: | ||
DevicePair(int parent, int device) | ||
: parent_(parent), | ||
device_(device) { | ||
} | ||
inline int parent() { | ||
return parent_; | ||
} | ||
inline int device() { | ||
return device_; | ||
} | ||
|
||
// Group GPUs in pairs, by proximity depending on machine's topology | ||
static void compute(const vector<int> devices, vector<DevicePair>* pairs); | ||
|
||
protected: | ||
int parent_; | ||
int device_; | ||
}; | ||
|
||
// Synchronous data parallelism using map-reduce between local GPUs. | ||
template<typename Dtype> | ||
class P2PSync : public GPUParams<Dtype>, public Solver<Dtype>::Callback, | ||
public InternalThread { | ||
public: | ||
explicit P2PSync(shared_ptr<Solver<Dtype> > root_solver, | ||
P2PSync<Dtype>* parent, const SolverParameter& param); | ||
virtual ~P2PSync(); | ||
|
||
inline const shared_ptr<Solver<Dtype> >& solver() const { | ||
return solver_; | ||
} | ||
|
||
void run(const vector<int>& gpus); | ||
|
||
protected: | ||
void on_start(); | ||
void on_gradients_ready(); | ||
|
||
void InternalThreadEntry(); | ||
|
||
P2PSync<Dtype>* parent_; | ||
vector<P2PSync<Dtype>*> children_; | ||
BlockingQueue<P2PSync<Dtype>*> queue_; | ||
const int initial_iter_; | ||
Dtype* parent_grads_; | ||
shared_ptr<Solver<Dtype> > solver_; | ||
|
||
using Params<Dtype>::size_; | ||
using Params<Dtype>::data_; | ||
using Params<Dtype>::diff_; | ||
}; | ||
|
||
} // namespace caffe | ||
|
||
#endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.