layr Namespace Reference

Classes
struct	Conv
	Layer implementation to apply conv2d functions to weight and optional bias. More...
struct	ConvBuilder
	Builder implementation for convolution layer. More...
struct	Dense
	Layer implementation to apply fully_connect functions to weight and optional bias. More...
struct	DenseBuilder
	Builder implementation for fully connected layer. More...
struct	iLayer
struct	iLayerBuilder
struct	LayerId
	Sublayer type, label, and index encapsulation. More...
struct	LayerRegistry
struct	LayerTag
	Tag implementation specifically for contents of layers. More...
struct	RBM
struct	RBMBuilder
	Builder implementation for restricted boltzmann layer. More...
struct	SeqModelBuilder
	Builder implementation for sequentially connected models. More...
struct	SequentialModel
	Layer implementation that sequentially applies sublayers. More...
struct	ULayer
	Layer implementation to apply activation and pooling functions. More...
struct	ULayerBuilder
	Builder implementation for activation layer. More...
struct	VarAssign
	Variable and error approximation assignment encapsulation. More...

Typedefs
using	ConvptrT = std::shared_ptr< Conv >
	Smart pointer of convolutional layer. More...
using	DenseptrT = std::shared_ptr< Dense >
	Smart pointer of fully connected layer. More...
using	VarErrsT = std::vector< std::pair< eteq::VarptrT< PybindT >, NodeptrT > >
	Ordered association between variable and error. More...
using	AssignsT = std::vector< VarAssign >
	One batch of assignments. More...
using	AssignGroupsT = std::vector< AssignsT >
	All batches of assignments. More...
using	ApproxF = std::function< AssignGroupsT(const VarErrsT &)>
using	UpdateStepF = std::function< void(teq::TensSetT &)>
template<typename T >
using	InitF = std::function< eteq::VarptrT< T >(teq::Shape, std::string)>
	Function that produces a variable given the variable's shape and label. More...
template<typename T >
using	ShapeFactorF = std::function< T(teq::Shape)>
	Function that returns some metric of a shape. More...
using	LayerIdsT = std::vector< LayerId >
	Vector of sublayer ids. More...
using	LayerptrT = std::shared_ptr< iLayer >
	Smart pointer of layer. More...
using	LBuilderptrT = std::shared_ptr< iLayerBuilder >
	Layer builder smart pointer. More...
using	LayerBuildF = std::function< LBuilderptrT(std::string)>
	Function that takes layer type and returns associated layer builder. More...
using	RBMptrT = std::shared_ptr< RBM >
	Smart pointer of RBM layer. More...
using	SeqModelptrT = std::shared_ptr< SequentialModel >
	Smart pointer of sequentially connected model. More...
using	UnaryF = std::function< NodeptrT(const ULayer &, NodeptrT)>
	Function that takes corresponding unary layer and node. More...
using	UnaryptrT = std::shared_ptr< ULayer >
	Smart pointer of unary layer. More...

Functions
AssignGroupsT	sgd (const VarErrsT &leaves, PybindT learning_rate=0.5, std::string root_label="")
AssignGroupsT	rms_momentum (const VarErrsT &leaves, PybindT learning_rate=0.5, PybindT discount_factor=0.99, PybindT epsilon=std::numeric_limits< PybindT >::epsilon(), std::string root_label="")
void	assign_groups (const AssignGroupsT &groups, UpdateStepF update_step)
	Apply all batches of assignments with update_step applied after each batch. More...
void	assign_groups_preupdate (const AssignGroupsT &groups, UpdateStepF update_step)
	Apply all batches of assignments with update_step applied before each batch. More...
template<typename T >
T	fanio (teq::Shape shape)
	Return the sum of the first 2 dimensions of a shape. More...
template<typename T >
T	fanavg (teq::Shape shape)
	Return the mean of the first 2 dimensions of a shape. More...
template<typename T >
void	truncated_normal (std::vector< T > &out, teq::Shape shape, T mean, T stdev, size_t max_repick=5)
template<typename T >
InitF< T >	zero_init (void)
	Return initialization function that makes zero variables. More...
template<typename T >
InitF< T >	variance_scaling_init (T factor, ShapeFactorF< T > sfactor=fanavg< T >)
template<typename T >
InitF< T >	unif_xavier_init (T factor=1)
template<typename T >
InitF< T >	norm_xavier_init (T factor=1)
void	validate_label (const std::string &label)
std::string	layer_label_fmt (std::string label, LayerId subid)
	Return formatted raw label with associated sublayer. More...
std::unordered_map< std::string, LayerIdsT >	unpack_labels (const std::vector< std::string > &labels)
	Return raw labels mapped to sublayers given a vector of formatted labels. More...
LayerRegistry &	get_layer_reg (void)
	Return global layer registry reference. More...
void	recursive_layer_tag (teq::TensptrT tens, std::string layer_type, std::string name, teq::TensSetT stops, LayerRegistry &registry=get_layer_reg())
LayerptrT	load_layer (std::istream &ins, teq::TensptrsT &roots, std::string ltype, std::string label, LayerRegistry &registry=get_layer_reg())
bool	save_layer (std::ostream &outs, const iLayer &layer, teq::TensptrsT roots, LayerRegistry &registry=get_layer_reg())
NodeptrT	softmax_from_layer (const ULayer &layer, NodeptrT input)
NodeptrT	maxpool_from_layer (const ULayer &layer, NodeptrT input)
NodeptrT	meanpool_from_layer (const ULayer &layer, NodeptrT input)
UnaryptrT	sigmoid (void)
	Return activation layer using sigmoid. More...
UnaryptrT	tanh (void)
	Return activation layer using tanh. More...
UnaryptrT	softmax (teq::RankT dim)
	Return activation layer using softmax of specified dimension. More...
UnaryptrT	maxpool2d (std::pair< teq::DimT, teq::DimT > dims={0, 1})
	Return pooling layer using max aggregation. More...
UnaryptrT	meanpool2d (std::pair< teq::DimT, teq::DimT > dims={0, 1})
	Return pooling layer using mean aggregation. More...

Variables
const std::string	conv_weight_key = "weight"
	Convolutional weight label. More...
const std::string	conv_bias_key = "bias"
	Convolutional bias label. More...
const std::string	conv_layer_key
	Identifier for convolutional layer. More...
const std::string	dense_weight_key = "weight"
	Fully connected weight label. More...
const std::string	dense_bias_key = "bias"
	Fully connected bias label. More...
const std::string	dense_layer_key
	Identifier for fully connected layer. More...
const std::string	layers_key_prefix = "layer_"
	String prefixed to every layer key. More...
const char	llabel_sep = ':'
	Layer label separator to divide each element in the LayerId. More...
const std::string	hidden_key = "hidden"
	Hidden fully connected layer label. More...
const std::string	visible_key = "visible"
	Visible fully connected layer label. More...
const std::string	rbm_layer_key
	Identifier for restricted boltzmann machine. More...
const std::string	seq_model_key
	Identifier for sequentially connected models. More...
const std::string	sigmoid_layer_key
	Identifier for sigmoid activation layer. More...
const std::string	tanh_layer_key
	Identifier for tanh activation layer. More...
const std::string	softmax_layer_key
	Identifier for softmax activation layer. More...
const std::string	maxpool2d_layer_key
	Identifier for max pooling layer. More...
const std::string	meanpool2d_layer_key
	Identifier for mean pooling layer. More...
const std::unordered_map< std::string, UnaryF >	unaries
	Map unary layer identifier to connection function. More...

Detailed Description

err_approx.hpp layr

Purpose: Define error approximation algorithms and variable assignment utilities

init.hpp layr

Purpose: Define variable initialization algorithms

layer.hpp layr

Purpose: Define layer interface and tagging

seqmodel.hpp layr

Purpose: Implement sequentially connected model

ulayer.hpp layr

Purpose: Implement generic layer that applies unary functions these functions don't store any data

Typedef Documentation

ApproxF

using layr::ApproxF = typedef std::function<AssignGroupsT(const VarErrsT&)>

Function that approximate error of sources given a vector of variables and its corresponding errors

AssignGroupsT

using layr::AssignGroupsT = typedef std::vector<AssignsT>

All batches of assignments.

AssignsT

using layr::AssignsT = typedef std::vector<VarAssign>

One batch of assignments.

ConvptrT

using layr::ConvptrT = typedef std::shared_ptr<Conv>

Smart pointer of convolutional layer.

DenseptrT

using layr::DenseptrT = typedef std::shared_ptr<Dense>

Smart pointer of fully connected layer.

InitF

template<typename T >

using layr::InitF = typedef std::function<eteq::VarptrT<T>(teq::Shape,std::string)>

Function that produces a variable given the variable's shape and label.

LayerBuildF

using layr::LayerBuildF = typedef std::function<LBuilderptrT(std::string)>

Function that takes layer type and returns associated layer builder.

LayerIdsT

using layr::LayerIdsT = typedef std::vector<LayerId>

Vector of sublayer ids.

LayerptrT

using layr::LayerptrT = typedef std::shared_ptr<iLayer>

Smart pointer of layer.

LBuilderptrT

using layr::LBuilderptrT = typedef std::shared_ptr<iLayerBuilder>

Layer builder smart pointer.

RBMptrT

using layr::RBMptrT = typedef std::shared_ptr<RBM>

Smart pointer of RBM layer.

SeqModelptrT

using layr::SeqModelptrT = typedef std::shared_ptr<SequentialModel>

Smart pointer of sequentially connected model.

ShapeFactorF

template<typename T >

using layr::ShapeFactorF = typedef std::function<T(teq::Shape)>

Function that returns some metric of a shape.

UnaryF

using layr::UnaryF = typedef std::function<NodeptrT( const ULayer&,NodeptrT)>

Function that takes corresponding unary layer and node.

UnaryptrT

using layr::UnaryptrT = typedef std::shared_ptr<ULayer>

Smart pointer of unary layer.

UpdateStepF

using layr::UpdateStepF = typedef std::function<void(teq::TensSetT&)>

Function that runs before or after variable assignment to calculate approximation graphs

VarErrsT

using layr::VarErrsT = typedef std::vector<std::pair<eteq::VarptrT<PybindT>,NodeptrT> >

Ordered association between variable and error.

Function Documentation

assign_groups()

void layr::assign_groups	(	const AssignGroupsT &	groups,
		UpdateStepF	update_step
	)

Apply all batches of assignments with update_step applied after each batch.

assign_groups_preupdate()

void layr::assign_groups_preupdate	(	const AssignGroupsT &	groups,
		UpdateStepF	update_step
	)

Apply all batches of assignments with update_step applied before each batch.

fanavg()

template<typename T >

T layr::fanavg

(

teq::Shape

shape

)

Return the mean of the first 2 dimensions of a shape.

fanio()

template<typename T >

T layr::fanio

(

teq::Shape

shape

)

Return the sum of the first 2 dimensions of a shape.

get_layer_reg()

LayerRegistry& layr::get_layer_reg

(

void

)

Return global layer registry reference.

layer_label_fmt()

std::string layr::layer_label_fmt	(	std::string	label,
		LayerId	subid
	)

Return formatted raw label with associated sublayer.

load_layer()

LayerptrT layr::load_layer	(	std::istream &	ins,
		teq::TensptrsT &	roots,
		std::string	ltype,
		std::string	label,
		LayerRegistry &	registry = get_layer_reg()
	)

Return a rebuilt layer from protobuf in stream (ins) a bunch of subgraph roots and the output layer's type and label

maxpool2d()

UnaryptrT layr::maxpool2d

(

std::pair< teq::DimT, teq::DimT >

dims = {0, 1}

)

Return pooling layer using max aggregation.

maxpool_from_layer()

NodeptrT layr::maxpool_from_layer	(	const ULayer &	layer,
		NodeptrT	input
	)

meanpool2d()

UnaryptrT layr::meanpool2d

(

std::pair< teq::DimT, teq::DimT >

dims = {0, 1}

)

Return pooling layer using mean aggregation.

meanpool_from_layer()

NodeptrT layr::meanpool_from_layer	(	const ULayer &	layer,
		NodeptrT	input
	)

norm_xavier_init()

template<typename T >

InitF<T> layr::norm_xavier_init

(

T

factor = 1

)

Return initialization function that makes xavier initialized variables (that uses gaussian distribution) (see https://andyljones.tumblr.com/post/110998971763/an-explanation-of-xavier-initialization)

recursive_layer_tag()

void layr::recursive_layer_tag	(	teq::TensptrT	tens,
		std::string	layer_type,
		std::string	name,
		teq::TensSetT	stops,
		LayerRegistry &	registry = get_layer_reg()
	)

Recursively tag tensor subgraph with specified layer type, and label only ignoring subgraphs of tensors in stops set

rms_momentum()

AssignGroupsT layr::rms_momentum	(	const VarErrsT &	leaves,
		PybindT	learning_rate = 0.5,
		PybindT	discount_factor = 0.99,
		PybindT	epsilon = std::numeric_limits<PybindT>::epsilon(),
		std::string	root_label = ""
	)

Return all batches of variable assignments of momentum-based rms error approximation applied to particular variables-error associations

Momentum-based Root Mean Square Approximation for each (x, err) in leaves momentum_next ~ χ * momentum_cur + (1 - χ) * err ^ 2 x_next ~ x_curr - (η * err) / (sqrt(ε + momentum_next))

where η is the learning rate, ε is epsilon, and χ is discount_factor initial momentum is 1

save_layer()

bool layr::save_layer	(	std::ostream &	outs,
		const iLayer &	layer,
		teq::TensptrsT	roots,
		LayerRegistry &	registry = get_layer_reg()
	)

Return true if specified layer and root subgraphs are saved to protobuf out stream (outs)

sgd()

AssignGroupsT layr::sgd	(	const VarErrsT &	leaves,
		PybindT	learning_rate = 0.5,
		std::string	root_label = ""
	)

Return all batches of variable assignments of stochastic gradient descent error approximation applied to particular variables-error associations

Stochastic Gradient Descent Approximation for each (x, err) in leaves x_next ~ x_curr - η * err,

where η is the learning rate

sigmoid()

UnaryptrT layr::sigmoid

(

void

)

Return activation layer using sigmoid.

softmax()

UnaryptrT layr::softmax

(

teq::RankT

dim

)

Return activation layer using softmax of specified dimension.

softmax_from_layer()

NodeptrT layr::softmax_from_layer	(	const ULayer &	layer,
		NodeptrT	input
	)

Softmax layer connection function that extracts transformation parameter from layer and apply to input

tanh()

UnaryptrT layr::tanh

(

void

)

Return activation layer using tanh.

truncated_normal()

template<typename T >

void layr::truncated_normal	(	std::vector< T > &	out,
		teq::Shape	shape,
		T	mean,
		T	stdev,
		size_t	max_repick = 5
	)

Populate out vector with normally distributed values (using mean and stdev) except repick values if the value is not within 2 stdev of the mean

unif_xavier_init()

template<typename T >

InitF<T> layr::unif_xavier_init

(

T

factor = 1

)

Return initialization function that makes xavier initialized variables (that uses uniform distribution) (see https://andyljones.tumblr.com/post/110998971763/an-explanation-of-xavier-initialization)

unpack_labels()

std::unordered_map<std::string,LayerIdsT> layr::unpack_labels

(

const std::vector< std::string > &

labels

)

Return raw labels mapped to sublayers given a vector of formatted labels.

validate_label()

void layr::validate_label

(

const std::string &

label

)

Check if the raw label does not contain llabel_sep as to not clash with LayerId representation

variance_scaling_init()

template<typename T >

InitF<T> layr::variance_scaling_init	(	T	factor,
		ShapeFactorF< T >	sfactor = fanavg<T>
	)

Return initialization function that makes variance scaling variables (see https://www.tensorflow.org/versions/r1.14/api_docs/python/tf/contrib/layers/variance_scaling_initializer)

zero_init()

template<typename T >

InitF<T> layr::zero_init

(

void

)

Return initialization function that makes zero variables.

Variable Documentation

conv_bias_key

const std::string layr::conv_bias_key = "bias"

Convolutional bias label.

conv_layer_key

const std::string layr::conv_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "conv",
[](std::string label) -> LBuilderptrT
{
    return std::make_shared<ConvBuilder>(label);
})

Identifier for convolutional layer.

conv_weight_key

const std::string layr::conv_weight_key = "weight"

Convolutional weight label.

dense_bias_key

const std::string layr::dense_bias_key = "bias"

Fully connected bias label.

dense_layer_key

const std::string layr::dense_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "dense",
[](std::string label) -> LBuilderptrT
{
    return std::make_shared<DenseBuilder>(label);
})

Identifier for fully connected layer.

dense_weight_key

const std::string layr::dense_weight_key = "weight"

Fully connected weight label.

hidden_key

const std::string layr::hidden_key = "hidden"

Hidden fully connected layer label.

layers_key_prefix

const std::string layr::layers_key_prefix = "layer_"

String prefixed to every layer key.

llabel_sep

const char layr::llabel_sep = ':'

Layer label separator to divide each element in the LayerId.

maxpool2d_layer_key

const std::string layr::maxpool2d_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "maxpool",
[](std::string extra_info) -> LBuilderptrT
{
    return std::make_shared<ULayerBuilder>(maxpool2d_layer_key, extra_info);
})

Identifier for max pooling layer.

meanpool2d_layer_key

const std::string layr::meanpool2d_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "meanpool",
[](std::string extra_info) -> LBuilderptrT
{
    return std::make_shared<ULayerBuilder>(meanpool2d_layer_key, extra_info);
})

Identifier for mean pooling layer.

rbm_layer_key

const std::string layr::rbm_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "rbm",
[](std::string label) -> LBuilderptrT
{
    return std::make_shared<RBMBuilder>(label);
})

Identifier for restricted boltzmann machine.

seq_model_key

const std::string layr::seq_model_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "seqmodel",
[](std::string label) -> LBuilderptrT
{
    return std::make_shared<SeqModelBuilder>(label);
})

Identifier for sequentially connected models.

sigmoid_layer_key

const std::string layr::sigmoid_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "sigmoid",
[](std::string extra_info) -> LBuilderptrT
{
    return std::make_shared<ULayerBuilder>(sigmoid_layer_key, "");
})

Identifier for sigmoid activation layer.

softmax_layer_key

const std::string layr::softmax_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "softmax",
[](std::string extra_info) -> LBuilderptrT
{
    return std::make_shared<ULayerBuilder>(softmax_layer_key, extra_info);
})

Identifier for softmax activation layer.

tanh_layer_key

const std::string layr::tanh_layer_key

Initial value:

=
get_layer_reg().register_tagr(layers_key_prefix + "tanh",
[](std::string extra_info) -> LBuilderptrT
{
    return std::make_shared<ULayerBuilder>(tanh_layer_key, "");
})

Identifier for tanh activation layer.

unaries

const std::unordered_map<std::string,UnaryF> layr::unaries

Initial value:

=
{
    {sigmoid_layer_key,
        [](const ULayer& layer, NodeptrT input)
        { return tenncor::sigmoid<PybindT>(input); }},
    {tanh_layer_key,
        [](const ULayer& layer, NodeptrT input)
        { return tenncor::tanh<PybindT>(input); }},
    {softmax_layer_key, softmax_from_layer},
    {maxpool2d_layer_key, maxpool_from_layer},
    {meanpool2d_layer_key, meanpool_from_layer},
}

Map unary layer identifier to connection function.

visible_key

const std::string layr::visible_key = "visible"

Visible fully connected layer label.