forked from hschwenk/cslm-toolkit
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMach.h
177 lines (161 loc) · 7.75 KB
/
Mach.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
/*
* This file is part of the continuous space language and translation model toolkit
* for statistical machine translation and large vocabulary speech recognition.
*
* Copyright 2015, Holger Schwenk, LIUM, University of Le Mans, France
*
* The CSLM toolkit is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License version 3 as
* published by the Free Software Foundation
*
* This library is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*
*
*/
#ifndef _Machine_h
#define _Machine_h
#include <iostream>
#include <fstream>
#include <map>
#include "Tools.h"
#include "Blas.h"
#include "Timer.h"
// list of all known machine types,
// this is needed for the general file read function
#define file_header_name "HPerf"
#define file_header_version1 1 // initial version
#define file_header_version2 2 // 2013/12/08: switched to ulong for nb_forw and nb_backw
#define file_header_version3 3 // 2015/03/18: added sharing information for MachTab
#define file_header_version4 4 // 2015/06/05: generalized sharing information: all simple machine can share its weights
#define file_header_version file_header_version4
#define file_header_size 16
#define file_header_mtype_base 0
#define file_header_mtype_tab 1
#define file_header_mtype_tabsh 2
#define file_header_mtype_lin 3
#define file_header_mtype_sig 4
#define file_header_mtype_tanh 5
#define file_header_mtype_softmax 6
#define file_header_mtype_stab 7
#define file_header_mtype_softmax_stable 8
#define file_header_mtype_lin_rectif 9
#define file_header_mtype_softmax_class 10
#define file_header_mtype_copy 11
#define file_header_mtype_multi 16
#define file_header_mtype_mseq 17
#define file_header_mtype_msplit1 18
#define file_header_mtype_mpar 19
#define file_header_mtype_msplit 20
#define file_header_mtype_mjoin 21
#define file_header_mtype_combined 31
#define file_header_mtype_max 32
#define file_header_mtype_avr 33
class Mach
{
private:
void do_alloc(); // perform allocation of dynamic data structures
protected:
static int fileid;
static std::map<int, Mach *> prSharedMachines; // to store Mach pointers for sharing using clone() function
int idim, odim; // input and output dimension
int bsize; // block size (nb of example used in parallel)
ulong nb_forw; // nb of forward examples processed
ulong nb_backw; // nb of backward examples processed
bool update; // update internal parameters
REAL lr_coeff; // machine specific learning coefficient (default 1.0)
// drop-out
REAL drop_out; // dropout probability, 0: not used (default), >0 apply drop-out (in training), <0 scale weighted sum (in testing)
REAL *drop_out_rand; // random values for the whole output vector
#if 0
// recurrent conncetions: user specified parameters
uint rec_hist; // nb of examples which are memorized
uint rec_step; // number of step before we do an update of the weights
uint rec_span; // number of step we go back during update
// rec_span can be larger than rec_step !
// both must be smaller or equal than rec_hist
// recurrent conncetions: for internal handling
// all the buffers are circular, no data is moved once stored
uint rec_ipos; // position in array where to add the new examples
// starts with 0, and wraps around once the end is reached
uint rec_len; // numnber of examples memorized in the buffers
#endif
// CUDA: the following four variables refer to device memory
// the size of these buffers is: DIM * bsize * rec_hist
REAL *data_in; // input data (pointer)
// CUDA: we need to allocate device memory
REAL *data_out; // output data (allocated by machine)
REAL *grad_in; // input gradients (allocated by machine)
REAL *grad_out; // output gradients (pointer)
// CUDA: we need to allocate device memory
Timer tm; // count real and user time
#ifdef BLAS_CUDA
size_t gpu_conf; // GPU configuration index; this is needed to run on multiple devices in parallel
#endif
// File or stream I/O, the following functions can be overloaded by subclass
// the main functions Read() and Write() should not be modified !
virtual void ReadParams(istream&, bool=true); // read all params
virtual void ReadData(istream&, size_t, int=0); // read binary data
virtual void WriteParams(ostream&); // write all params
virtual void WriteData(ostream&); // write binary data
Mach(const Mach &, const int=0); // create a copy of the machine
public:
Mach(const int=0, const int=0, const int=128, const ulong=0, const ulong=0);
virtual ~Mach();
virtual Mach *Clone() {return new Mach(*this);} // create a copy of the machine
// Tools
virtual int GetMType() {return file_header_mtype_base;}; // get type of machine
virtual int GetIdim() {return idim;}
int GetOdim() {return odim;}
int GetBsize() {return bsize;}
virtual void SetBsize(int bs) {
if (bs<1) Error("wrong value in SetBsize()"); else bsize=bs; }
ulong GetNbForw() {return nb_forw;}
ulong GetNbBackw() {return nb_backw;}
virtual void SetNbEx(ulong nf, ulong nb) {nb_forw=nf; nb_backw=nb;}
virtual ulong GetNbParams() {return 0;} // return the nbr of allocated parameters
void SetUpdataParams(bool up) {update=up;} // change flag to update internal parameters
void SetLrateCoeff(REAL v) {lr_coeff=v;}
virtual REAL* GetDataIn() {return data_in;} // return pointer on input data for chaining
virtual REAL* GetDataOut() {return data_out;} // return pointer on output data for chaining
virtual REAL* GetGradIn() {return grad_in;} // return pointer on input gradient for chaining
virtual REAL* GetGradOut() {return grad_out;} // return pointer on output gradient for chaining
virtual void SetDataIn(REAL *data) {data_in=data;} // set pointer of input data
virtual void SetGradOut(REAL *data) {grad_out=data;} // set pointer of output gradient
virtual void SetDropOut(const REAL v); // set drop-out fraction
#ifdef BLAS_CUDA
size_t GetGpuConfig() { return gpu_conf; } // return GPU configuration index used for this machine
#endif
virtual void Info(bool=false, char *txt=(char*)" - ");// display (detailed) information on machine
virtual bool CopyParams(Mach*); // copy parameters from another machine
// FILE IO
static Mach *Read(istream&, int=0); // read class from a stream
static Mach *ReadFromFile(const char*, int=0); // read class from a file, call Read
void Write(ostream&); // write content of class to a stream
void WriteToFile(const char*); // write content of class to a stream
// Training
virtual void Forw(int=0, bool=false); // calculate outputs for current inputs
// backprop gradients from output to input and update all weights
virtual void Backw (const float lrate, const float wdecay, int =0);
static int GetFileId(){ return fileid;}
static void SetFileId(int id){ fileid = id;}
static Mach* GetSharedMachine(int i){ return prSharedMachines[i]; }
static void SetSharedMachine(int i, Mach* m){ prSharedMachines[i]=m; }
static bool canShare(int mtype) {
return (mtype != file_header_mtype_base
&& mtype != file_header_mtype_stab
&& mtype <= file_header_mtype_softmax_class);
}
static void ResetSharedMachines() { prSharedMachines.clear(); }
};
void GpuUnlock();
// Find sub-machines matching desired mtype in parent_mach (depth-first).
Mach* FindFirstMatching(int mtype, Mach* parent_mach);
std::vector<Mach*> FindAllMatching(int mtype, Mach* parent_mach);
#endif