Tenncor
grader.hpp
Go to the documentation of this file.
1 
9 #include <list>
10 
11 #include "teq/grad_def.hpp"
12 
13 #include "eteq/generated/api.hpp"
14 
15 #include "eteq/constant.hpp"
16 
17 #ifndef ETEQ_GRADER_HPP
18 #define ETEQ_GRADER_HPP
19 
20 namespace eteq
21 {
22 
24 template <typename T>
25 NodeptrT<T> reduce_grad (const teq::FuncArg& child,
26  NodeptrT<T> bwd, size_t idx)
27 {
28  const teq::Shape& shape = child.get_tensor()->shape();
29  teq::CoordptrT revshaper(child.get_shaper()->reverse());
30  CoordptrT revcoord;
31  {
32  auto coorder = child.get_coorder();
33  assert(nullptr != coorder);
34  teq::CoordT dims;
35  coorder->forward(dims.begin(), dims.begin());
36  teq::CoordT bcast;
37  std::fill(bcast.begin(), bcast.end(), 1);
38  for (teq::RankT d : dims)
39  {
40  if (d < teq::rank_cap)
41  {
42  bcast[d] = shape.at(d);
43  }
44  }
45  revcoord = std::make_shared<CoordMap>(bcast, false);
46  }
47  return make_functor<T>(teq::Opcode{"EXTEND",egen::EXTEND}, {
48  FuncArg<T>(bwd, revshaper, revcoord)
49  });
50 }
51 
53 template <typename T>
54 NodeptrT<T> permute_grad (teq::iFunctor* fwd,
55  NodeptrT<T> bwd, size_t idx)
56 {
57  const auto& child = fwd->get_children()[0];
58  teq::CoordptrT revshaper(child.get_shaper()->reverse());
59  CoordptrT revcoord;
60  {
61  auto coorder = child.get_coorder();
62  assert(nullptr != coorder);
63  teq::CoordT dims;
64  coorder->forward(dims.begin(), dims.begin());
65 
66  teq::CoordT order;
67  for (teq::RankT i = 0; i < teq::rank_cap; ++i)
68  {
69  order[dims[i]] = i;
70  }
71  revcoord = std::make_shared<CoordMap>(order, true);
72  }
73  return make_functor<T>(teq::Opcode{"PERMUTE",egen::PERMUTE},{
74  FuncArg<T>(bwd, revshaper, revcoord)
75  });
76 }
77 
79 template <typename T>
80 NodeptrT<T> extend_grad (teq::iFunctor* fwd,
81  NodeptrT<T> bwd, size_t idx)
82 {
83  const auto& child = fwd->get_children()[0];
84  teq::CoordptrT revshaper(child.get_shaper()->reverse());
85  CoordptrT revcoord;
86  {
87  auto coorder = child.get_coorder();
88  assert(nullptr != coorder);
89  teq::CoordT dims;
90  coorder->forward(dims.begin(), dims.begin());
91  std::vector<teq::RankT> red_dims;
92  for (teq::RankT i = 0; i < teq::rank_cap; ++i)
93  {
94  if (dims[i] > 1)
95  {
96  red_dims.push_back(i);
97  }
98  }
99  revcoord = reduce(red_dims);
100  }
101  return make_functor<T>(teq::Opcode{"REDUCE_SUM",egen::REDUCE_SUM},{
102  FuncArg<T>(bwd, revshaper, revcoord)
103  });
104 }
105 
107 template <typename T>
108 struct GradientBuilder final : public teq::iGradientBuilder
109 {
111  teq::TensptrT local_derivative (teq::FuncptrT op,
112  size_t arg_idx) const override
113  {
114  const teq::ArgsT& args = op->get_children();
115  NodeptrT<T> out = nullptr;
116  teq::Opcode opcode = op->get_opcode();
117  switch ((egen::_GENERATED_OPCODE) opcode.code_)
118  {
119  case egen::ABS:
120  out = TO_NODE(args[0].get_tensor()) / TO_NODE(op);
121  break;
122  case egen::NEG:
123  out = make_constant_scalar<T>(
124  -1, args[0].get_tensor()->shape());
125  break;
126  case egen::SIN:
127  out = tenncor::cos(TO_NODE(args[0].get_tensor()));
128  break;
129  case egen::COS:
130  out = -tenncor::sin(TO_NODE(args[0].get_tensor()));
131  break;
132  case egen::TAN:
133  out = (T) 1 / tenncor::pow(
134  tenncor::cos(TO_NODE(args[0].get_tensor())), (T) 2);
135  break;
136  case egen::EXP:
137  out = TO_NODE(op);
138  break;
139  case egen::LOG:
140  out = (T) 1 / TO_NODE(args[0].get_tensor());
141  break;
142  case egen::SQRT:
143  out = (T) 1 / ((T) 2 * TO_NODE(op));
144  break;
145  case egen::SQUARE:
146  out = (T) 2 * TO_NODE(args[0].get_tensor());
147  break;
148  case egen::CUBE:
149  out = (T) 3 * tenncor::square(TO_NODE(args[0].get_tensor()));
150  break;
151  case egen::SIGMOID:
152  out = tenncor::sigmoid_grad(
153  TO_NODE(args[0].get_tensor()));
154  break;
155  case egen::SIGMOID_GRAD:
156  out = TO_NODE(op) * ((T) 1 - (T) 2 *
157  tenncor::sigmoid(TO_NODE(args[0].get_tensor())));
158  break;
159  case egen::TANH:
160  out = (T) 1 - tenncor::square(TO_NODE(op));
161  break;
162  case egen::ROUND:
163  case egen::REDUCE_SUM:
164  case egen::EXTEND:
165  case egen::PERMUTE:
166  case egen::ADD:
167  case egen::SLICE:
168  case egen::PAD:
169  case egen::STRIDE: // todo: figure out if this belongs here
170  out = make_constant_scalar<T>(1, args[0].get_tensor()->shape());
171  break;
172  case egen::MUL:
173  case egen::CONV:
174  out = TO_NODE(args[(size_t)(arg_idx==0)].get_tensor());
175  break;
176  case egen::MAX:
177  case egen::MIN:
178  out = TO_NODE(op) == TO_NODE(args[arg_idx].get_tensor());
179  break;
180  case egen::POW:
181  out = arg_idx==0 ?
182  TO_NODE(args[1].get_tensor()) *
183  tenncor::pow(
184  TO_NODE(args[0].get_tensor()),
185  TO_NODE(args[1].get_tensor()) - (T) 1
186  ) :
187  tenncor::log(TO_NODE(args[0].get_tensor())) *
188  TO_NODE(op);
189  break;
190  case egen::SUB:
191  out = make_constant_scalar<T>(arg_idx == 0 ?
192  1 : -1, args[0].get_tensor()->shape());
193  break;
194  case egen::DIV:
195  {
196  auto denom = TO_NODE(args[1].get_tensor());
197  out = arg_idx==0 ?
198  (T) 1 / denom :
199  -TO_NODE(args[0].get_tensor()) / denom / denom;
200  }
201  break;
202  case egen::EQ:
203  case egen::NEQ:
204  case egen::GT:
205  case egen::LT:
206  case egen::RAND_UNIF:
207  case egen::SELECT:
208  out = make_constant_scalar<T>(0, args[0].get_tensor()->shape());
209  break;
210  case egen::REDUCE_PROD: // todo: prevent divide by zero
211  out =
212  reduce_grad(args[0], TO_NODE(op), arg_idx) /
213  TO_NODE(args[0].get_tensor());
214  break;
215  case egen::REDUCE_MAX:
216  case egen::REDUCE_MIN:
217  out =
218  reduce_grad(args[0], TO_NODE(op), arg_idx) ==
219  TO_NODE(args[0].get_tensor());
220  break;
221  case egen::MATMUL:
222  {
223  NodeptrT<T> lhs = TO_NODE(args[0].get_tensor());
224  NodeptrT<T> rhs = TO_NODE(args[1].get_tensor());
225  out = 0 == arg_idx ?
226  // ext_rhs
227  tenncor::permute(tenncor::extend(rhs, 2, {
228  lhs->shape().at(1)}), {0,2,1}) :
229  // ext_lhs
230  tenncor::permute(tenncor::extend(lhs, 2, {
231  rhs->shape().at(0)}), {2,1,0});
232  }
233  break;
234  case egen::CONV_IMG_GRAD:
235  logs::fatal("cannot derive CONV_IMG_GRAD");
236  break;
237  case egen::CONV_KRN_GRAD:
238  logs::fatal("cannot derive CONV_KRN_GRAD");
239  break;
240  default:
241  logs::fatalf("Unknown op %s", opcode.name_.c_str());
242  }
243  return out->get_tensor();
244  }
245 
247  teq::TensptrT chain_rule (teq::FuncptrT op, const teq::TensptrT& local_der,
248  teq::TensptrT supcomp_grad, size_t arg_idx) const override
249  {
250  NodeptrT<T> out = nullptr;
251  teq::Opcode opcode = op->get_opcode();
252  switch (opcode.code_)
253  {
254  case egen::ABS:
255  case egen::NEG:
256  case egen::SIN:
257  case egen::COS:
258  case egen::TAN:
259  case egen::EXP:
260  case egen::LOG:
261  case egen::SQRT:
262  case egen::SQUARE:
263  case egen::CUBE:
264  case egen::ROUND:
265  case egen::SIGMOID:
266  case egen::SIGMOID_GRAD:
267  case egen::TANH:
268  case egen::ADD:
269  case egen::MUL:
270  case egen::MAX:
271  case egen::MIN:
272  case egen::POW:
273  case egen::SUB:
274  case egen::DIV:
275  case egen::EQ:
276  case egen::NEQ:
277  case egen::GT:
278  case egen::LT:
279  case egen::RAND_UNIF:
280  out = TO_NODE(local_der) *
281  TO_NODE(supcomp_grad);
282  break;
283  case egen::REDUCE_MAX:
284  case egen::REDUCE_MIN:
285  case egen::REDUCE_PROD:
286  case egen::REDUCE_SUM:
287  out = TO_NODE(local_der) * reduce_grad(
288  op->get_children()[0], TO_NODE(supcomp_grad), arg_idx);
289  break;
290  case egen::EXTEND:
291  out = TO_NODE(local_der) * extend_grad(
292  op.get(), TO_NODE(supcomp_grad), arg_idx);
293  break;
294  case egen::PERMUTE:
295  out = TO_NODE(local_der) * permute_grad(
296  op.get(), TO_NODE(supcomp_grad), arg_idx);
297  break;
298  case egen::MATMUL:
299  out = tenncor::reduce_sum(
300  tenncor::permute(
301  TO_NODE(local_der) *
302  tenncor::extend(TO_NODE(supcomp_grad), 2, {
303  op->get_children()[0].
304  get_tensor()->shape().at(0)
305  }),
306  0 == arg_idx ?
307  std::vector<teq::RankT>{2, 1, 0} :
308  std::vector<teq::RankT>{0, 2, 1}), 2, 1);
309  break;
310  case egen::CONV:
311  {
312  teq::Opcode opcode;
313  auto args = op->get_children();
314  teq::CoordptrT fwd_shaper =
315  args[(size_t)(0 == arg_idx)].get_shaper();
316  teq::CoordptrT rev_shaper(
317  args[arg_idx].get_shaper()->reverse());
318  if (arg_idx == 0)
319  {
320  opcode = teq::Opcode{"CONV_IMG_GRAD",
321  egen::CONV_IMG_GRAD};
322  }
323  else
324  {
325  opcode = teq::Opcode{"CONV_KRN_GRAD",
326  egen::CONV_KRN_GRAD};
327  }
328  teq::CoordptrT full_shaper(
329  fwd_shaper->connect(*rev_shaper));
330  out = make_functor<T>(opcode, {
331  FuncArg<T>(TO_NODE(local_der), full_shaper, nullptr),
332  FuncArg<T>(TO_NODE(supcomp_grad), rev_shaper, nullptr),
333  });
334  }
335  break;
336  case egen::SLICE:
337  {
338  teq::CoordT slicings;
339  auto& child = op->get_children()[0];
340  child.get_coorder()->forward(
341  slicings.begin(), slicings.begin());
342  teq::DimT dimension = slicings[2];
343  teq::DimT dim = child.get_tensor()->shape().at(dimension);
344  teq::DimT left_pad = slicings[0];
345  teq::DimT right_pad = dim - (left_pad + slicings[1]);
346  out = TO_NODE(local_der) *
347  tenncor::pad(TO_NODE(supcomp_grad),
348  std::pair<teq::DimT,teq::DimT>{
349  left_pad, right_pad}, dimension);
350  }
351  break;
352  case egen::PAD:
353  {
354  teq::CoordT paddings;
355  auto& child = op->get_children()[0];
356  child.get_coorder()->forward(
357  paddings.begin(), paddings.begin());
358  teq::DimT dimension = paddings[2];
359  teq::DimT dim = op->shape().at(dimension);
360  teq::DimT offset = paddings[0];
361  teq::DimT extent = dim - paddings[1] - offset;
362  out = TO_NODE(local_der) *
363  tenncor::slice(TO_NODE(supcomp_grad),
364  offset, extent, dimension);
365  }
366  break;
367  case egen::SELECT:
368  {
369  if (0 == arg_idx)
370  {
371  out = TO_NODE(local_der);
372  break;
373  }
374  auto condition = TO_NODE(
375  op->get_children()[0].get_tensor());
376  auto then = TO_NODE(supcomp_grad);
377  auto otherwise = make_constant_scalar<T>(0, op->shape());
378  if (1 < arg_idx)
379  {
380  std::swap(then, otherwise);
381  }
382  out = tenncor::if_then_else(condition, then, otherwise);
383  }
384  break;
385  case egen::CONV_IMG_GRAD:
386  logs::fatal("cannot derive CONV_IMG_GRAD");
387  break;
388  case egen::CONV_KRN_GRAD:
389  logs::fatal("cannot derive CONV_KRN_GRAD");
390  break;
391  case egen::STRIDE: // todo: implement
392  default:
393  logs::fatalf("Unknown op %s", opcode.name_.c_str());
394  }
395  return out->get_tensor();
396  }
397 
399  teq::TensptrT get_const_one (teq::Shape shape) const override
400  {
401  return make_constant_scalar<T>(1, shape)->get_tensor();
402  }
403 
405  teq::TensptrT get_const_zero (teq::Shape shape) const override
406  {
407  return make_constant_scalar<T>(0, shape)->get_tensor();
408  }
409 
411  teq::TensptrT add (teq::TensptrT& lhs, teq::TensptrT& rhs) const override
412  {
413  return teq::TensptrT(Functor<T>::get(teq::Opcode{"ADD", egen::ADD}, {
414  identity_map(TO_NODE(lhs)),
415  identity_map(TO_NODE(rhs))
416  }));
417  }
418 };
419 
421 template <typename T>
422 NodeptrT<T> derive (NodeptrT<T> root, NodeptrT<T> target)
423 {
424  GradientBuilder<T> builder;
425  teq::TensptrT derivative = builder.derive(
426  root->get_tensor(), target->get_tensor());
427  return TO_NODE(derivative);
428 }
429 
430 }
431 
432 #endif // ETEQ_GRADER_HPP
teq::CoordT
std::array< CDimT, rank_cap > CoordT
Definition: shape.hpp:56
teq::rank_cap
const RankT rank_cap
Number of dimsensions in a shape/coordinate.
Definition: shape.hpp:47
eteq::derive
NodeptrT< T > derive(NodeptrT< T > root, NodeptrT< T > target)
Derive root with respect to target and optimized.
Definition: grader.hpp:422
csv_to_png.args
args
Definition: csv_to_png.py:105
eteq::sin
EigenptrT< T > sin(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:280
teq::iFunctor::get_children
virtual const ArgsT & get_children(void) const =0
Return children nodes as a vector of raw pointers.
eteq::reduce_grad
NodeptrT< T > reduce_grad(const teq::FuncArg &child, NodeptrT< T > bwd, size_t idx)
Return reduction operator gradient of reduced functor node (bwd)
Definition: grader.hpp:25
teq::Opcode
Encoding of operation.
Definition: ifunctor.hpp:18
eteq::extend
CoordptrT extend(teq::RankT rank, std::vector< teq::DimT > ext)
Return CoordMap wrapper of extension parameters.
teq::FuncArg::get_shaper
CoordptrT get_shaper(void) const
Return shaper coord map.
Definition: funcarg.hpp:67
teq::iFunctor
Interface of iOperation-defined operation node.
Definition: ifunctor.hpp:28
eteq::square
EigenptrT< T > square(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:559
eteq
Definition: constant.hpp:17
teq::ArgsT
std::vector< FuncArg > ArgsT
Type of functor arguments.
Definition: funcarg.hpp:101
eteq::reduce
CoordptrT reduce(std::vector< teq::RankT > red_dims)
Return CoordMap wrapper of reduction dimensions.
teq::RankT
uint8_t RankT
Type used for shape rank.
Definition: shape.hpp:23
teq::Shape
Definition: shape.hpp:62
eteq::FuncArg
Eigen node version of teq::FuncArg.
Definition: funcarg.hpp:22
teq::FuncArg::get_tensor
TensptrT get_tensor(void) const
Return tensor being mapped.
Definition: funcarg.hpp:61
teq::CoordptrT
std::shared_ptr< iCoordMap > CoordptrT
Type of iCoordMap smartpointer.
Definition: coord.hpp:106
eteq::extend_grad
NodeptrT< T > extend_grad(teq::iFunctor *fwd, NodeptrT< T > bwd, size_t idx)
Return extension gradient of extended functor node (bwd)
Definition: grader.hpp:80
eteq::GradientBuilder::chain_rule
teq::TensptrT chain_rule(teq::FuncptrT op, const teq::TensptrT &local_der, teq::TensptrT supcomp_grad, size_t arg_idx) const override
Implementation of iGradientBuilder.
Definition: grader.hpp:247
teq::Opcode::name_
std::string name_
String representation of operation.
Definition: ifunctor.hpp:21
eteq::GradientBuilder::get_const_one
teq::TensptrT get_const_one(teq::Shape shape) const override
Implementation of iGradientBuilder.
Definition: grader.hpp:399
eteq::GradientBuilder::add
teq::TensptrT add(teq::TensptrT &lhs, teq::TensptrT &rhs) const override
Implementation of iGradientBuilder.
Definition: grader.hpp:411
teq::FuncArg::get_coorder
CoordptrT get_coorder(void) const
Return coord map for coordinates.
Definition: funcarg.hpp:80
eteq::CoordptrT
std::shared_ptr< CoordMap > CoordptrT
Type of iCoordMap smartpointer.
Definition: coord.hpp:64
teq::FuncArg
Coordinate mapper and tensor pair.
Definition: funcarg.hpp:21
eteq::permute_grad
NodeptrT< T > permute_grad(teq::iFunctor *fwd, NodeptrT< T > bwd, size_t idx)
Return permutation gradient of permuted functor node (bwd)
Definition: grader.hpp:54
eteq::GradientBuilder::get_const_zero
teq::TensptrT get_const_zero(teq::Shape shape) const override
Implementation of iGradientBuilder.
Definition: grader.hpp:405
teq::iGradientBuilder
Definition: grad_def.hpp:28
eteq::sigmoid_grad
EigenptrT< T > sigmoid_grad(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:498
eteq::sigmoid
EigenptrT< T > sigmoid(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:475
grad_def.hpp
teq::Opcode::code_
size_t code_
Numerical encoding of operation.
Definition: ifunctor.hpp:24
eteq::permute
CoordptrT permute(std::vector< teq::RankT > dims)
Return CoordMap wrapper of permute indices.
eteq::slice
EigenptrT< T > slice(teq::Shape &outshape, const OpArg< T > &in)
Return Eigen data object representing data slicing of dimensions.
Definition: operator.hpp:157
teq::TensptrT
std::shared_ptr< iTensor > TensptrT
Tensor smart pointer.
Definition: itensor.hpp:51
teq::DimT
uint16_t DimT
Type used for shape dimension.
Definition: shape.hpp:31
eteq::pad
EigenptrT< T > pad(teq::Shape &outshape, const OpArg< T > &in)
Return Eigen data object representing data zero padding.
Definition: operator.hpp:183
eteq::cos
EigenptrT< T > cos(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:314
eteq::GradientBuilder
ETEQ implementation of TEQ&#39;s Backward Propagation Builder.
Definition: grader.hpp:108
eteq::GradientBuilder::local_derivative
teq::TensptrT local_derivative(teq::FuncptrT op, size_t arg_idx) const override
Implementation of iGradientBuilder.
Definition: grader.hpp:111
eteq::log
EigenptrT< T > log(teq::Shape &outshape, const OpArg< T > &in)
Definition: operator.hpp:402
eteq::reduce_sum
EigenptrT< T > reduce_sum(teq::Shape &outshape, const OpArg< T > &in) template< typename T > EigenptrT< T > reduce_prod(teq
Return Eigen data object representing reduction where aggregation is sum.
Definition: operator.hpp:94
eteq::Functor
Functor implementation of operable functor of Eigen operators.
Definition: functor.hpp:25
eteq::NodeptrT
std::shared_ptr< iNode< T > > NodeptrT
Smart pointer of node.
Definition: inode.hpp:63
teq::FuncptrT
std::shared_ptr< iFunctor > FuncptrT
Functor smart pointer.
Definition: ifunctor.hpp:49
eteq::identity_map
FuncArg< T > identity_map(NodeptrT< T > node)
Return FuncArg<T> that identity maps input tensor.
Definition: funcarg.hpp:88
teq::Shape::at
DimT at(RankT idx) const
Return DimT element at idx for any index in range [0:rank_cap)
Definition: shape.hpp:108
teq::iGradientBuilder::derive
TensptrT derive(TensptrT root, TensptrT target) const
Return derivative of root with respect to target.
Definition: grad_def.hpp:54
TO_NODE
#define TO_NODE(tens)
Macro for converting tensor to node.
Definition: inode.hpp:106
eteq::pow
EigenptrT< T > pow(teq::Shape &outshape, const OpArg< T > &a, const OpArg< T > &b)
Definition: operator.hpp:608
Generated by   doxygen 1.8.14