# pylearn2 tutorial example: cifar_grbm_smd.yaml by Ian Goodfellow

#

# Read the README file before reading this file

#

# This is an example of yaml file, which is the main way that an experimenter

# interacts with pylearn2.

#

# A yaml file is very similar to a python dictionary, with a bit of extra

# syntax.

# The !obj tag allows us to create a specific class of object. The text after

# the : indicates what class should be loaded. This is followed by a pair of

# braces containing the arguments to that class's __init__ method.

#

# Here, we allocate a Train object, which represents the main loop of the

# training script. The train script will run this loop repeatedly. Each time

# through the loop, the model is trained on data from a training dataset, then

# saved to file.

!obj:pylearn2.train.Train {

    # The !pkl tag is used to create an object from a pkl file. Here we retrieve

    # the dataset made by make_dataset.py and use it as our training dataset.

    dataset: !pkl: "cifar10_preprocessed_train.pkl",

    # Next we make the model to be trained. It is a Binary Gaussian RBM

    model: !obj:pylearn2.models.rbm.GaussianBinaryRBM {

        # The RBM needs 192 visible units (its inputs are 8x8 patches with 3

        # color channels)

        nvis : 192,

        # We'll use 400 hidden units for this RBM. That's a small number but we

        # want this example script to train quickly.

        nhid : 400,

        # The elements of the weight matrices of the RBM will be drawn

        # independently from U(-0.05, 0.05)

        irange : 0.05,

        # There are many ways to parameterize a GRBM. Here we use a

        # parameterization that makes the correspondence to denoising

        # autoencoders more clear.

        energy_function_class : !obj:pylearn2.energy_functions.rbm_energy.grbm_type_1 {},

        # Some learning algorithms are capable of estimating the standard

        # deviation of the visible units of a GRBM successfully, others are not

        # and just fix the standard deviation to 1.  We're going to show off

        # and learn the standard deviation.

        learn_sigma : True,

        # Learning works better if we provide a smart initialization for the

        # parameters.  Here we start sigma at .4 , which is about the same

        # standard deviation as the training data. We start the biases on the

        # hidden units at -2, which will make them have fairly sparse

        # activations.

        init_sigma : .4,

        init_bias_hid : -2.,

        # Some GRBM training algorithms can't handle the visible units being

        # noisy and just use their mean for all computations. We will show off

        # and not use that hack here.

        mean_vis : False,

        # One hack we will make is we will scale back the gradient steps on the

        # sigma parameter. This way we don't need to worry about sigma getting

        # too small prematurely (if it gets too small too fast the learning

        # signal gets weak).

        sigma_lr_scale : 1e-3

    },

    # Next we need to specify the training algorithm that will be used to train

    # the model.  Here we use stochastic gradient descent.

    algorithm: !obj:pylearn2.training_algorithms.sgd.SGD {

        # The learning rate determines how big of steps the learning algorithm

        # takes.  Here we use fairly big steps initially because we have a

        # learning rate adjustment scheme that will scale them down if

        # necessary.

        learning_rate : 1e-1,

        # Each gradient step will be based on this many examples

        batch_size : 5,

        # We'll monitor our progress by looking at the first 20 batches of the

        # training dataset. This is an estimate of the training error. To be

        # really exhaustive, we could use the entire training set instead,

        # or to avoid overfitting, we could use held out data instead.

        monitoring_batches : 20,

        monitoring_dataset : !pkl: "cifar10_preprocessed_train.pkl",

        # Here we specify the objective function that stochastic gradient

        # descent should minimize.  In this case we use denoising score

        # matching, which makes this RBM behave as a denoising autoencoder.

        # See

        # Pascal Vincent. "A Connection Between Score Matching and Denoising

        # Auutoencoders." Neural Computation, 2011

        # for details.

        cost : !obj:pylearn2.costs.ebm_estimation.SMD {

            # Denoising score matching uses a corruption process to transform

            # the raw data.  Here we use additive gaussian noise.

            corruptor : !obj:pylearn2.corruption.GaussianCorruptor {

                    stdev : 0.4

            },

        },

        # We'll use the monitoring dataset to figure out when to stop training.

        #

        # In this case, we stop if there is less than a 1% decrease in the

        # training error in the last epoch.  You'll notice that the learned

        # features are a bit noisy. If you'd like nice smooth features you can

        # make this criterion stricter so that the model will train for longer.

        # (setting N to 10 should make the weights prettier, but will make it

        # run a lot longer)

        termination_criterion : !obj:pylearn2.termination_criteria.MonitorBased {

            prop_decrease : 0.01,

            N : 1,

        },

        # Let's throw a learning rate adjuster into the training algorithm.

        # To do this we'll use an "extension," which is basically an event

        # handler that can be registered with the Train object.

        # This particular one is triggered on each epoch.

        # It will shrink the learning rate if the objective goes up and increase

        # the learning rate if the objective decreases too slowly. This makes

        # our learning rate hyperparameter less important to get right.

        # This is not a very mathematically principled approach, but it works

        # well in practice.

        },

    extensions : [!obj:pylearn2.training_algorithms.sgd.MonitorBasedLRAdjuster {}],

    #Finally, request that the model be saved after each epoch

    save_freq : 1

}

#!/usr/bin/env python

"""

Script implementing the logic for training pylearn2 models.

This is intended to be a "driver" for most training experiments. A user

specifies an object hierarchy in a configuration file using a dictionary-like

syntax and this script takes care of the rest.

For example configuration files that are consumable by this script, see

    pylearn2/scripts/train_example

    pylearn2/scripts/autoencoder_example

"""

__authors__ = "Ian Goodfellow"

__copyright__ = "Copyright 2010-2012, Universite de Montreal"

__credits__ = ["Ian Goodfellow", "David Warde-Farley"]

__license__ = "3-clause BSD"

__maintainer__ = "Ian Goodfellow"

__email__ = "goodfeli@iro"

# Standard library imports

import argparse

import gc

import logging

import os

# Third-party imports

import numpy as np

# Local imports

from pylearn2.utils import serial

from pylearn2.utils.logger import (

    CustomStreamHandler, CustomFormatter, restore_defaults

)

class FeatureDump(object):

    def __init__(self, encoder, dataset, path, batch_size=None, topo=False):

        self.encoder = encoder

        self.dataset = dataset

        self.path = path

        self.batch_size = batch_size

        self.topo = topo

    def main_loop(self):

        if self.batch_size is None:

            if self.topo:

                data = self.dataset.get_topological_view()

            else:

                data = self.dataset.get_design_matrix()

            output = self.encoder.perform(data)

        else:

            myiterator = self.dataset.iterator(mode='sequential',

                                               batch_size=self.batch_size,

                                               topo=self.topo)

            chunks = []

            for data in myiterator:

                chunks.append(self.encoder.perform(data))

            output = np.concatenate(chunks)

        np.save(self.path, output)

def make_argument_parser():

    parser = argparse.ArgumentParser(

        description="Launch an experiment from a YAML configuration file.",

        epilog='\n'.join(__doc__.strip().split('\n')[1:]).strip(),

        formatter_class=argparse.RawTextHelpFormatter

    ) #parser是用来接收参数的

    parser.add_argument('--level-name', '-L',

                        action='store_true',

                        help='Display the log level (e.g. DEBUG, INFO) '

                             'for each logged message')

    parser.add_argument('--timestamp', '-T',

                        action='store_true',

                        help='Display human-readable timestamps for '

                             'each logged message')

    parser.add_argument('--verbose-logging', '-V',

                        action='store_true',

                        help='Display timestamp, log level and source '

                             'logger for every logged message '

                             '(implies -T).')

    parser.add_argument('--debug', '-D',

                        action='store_true',

                        help='Display any DEBUG-level log messages, '

                             'suppressed by default.')

    parser.add_argument('config', action='store', #按照格式输入参数，比如这里的输入的参数会保存在config中

                        choices=None,

                        help='A YAML configuration file specifying the '

                             'training procedure')

    return parser

if __name__ == "__main__":

    parser = make_argument_parser()

    args = parser.parse_args() #读取传入进来的参数，这里是直接在命令行读取该文件,参数放入args.config中

    train_obj = serial.load_train_file(args.config) #serial.load_train_file()函数最后返回的是：
　　　　　　　　　　# return yaml_parse.load_path(args.config) 也就是说调用的是ymal_parse.load_path()函数。返回的是一个train类的对象。
　　　　　　　　　　# 其中的ymal_parse是pylearn2.config中的函数。

return yaml_parse.load_path(config_file_path)

    try:

        iter(train_obj) #iter()是个迭代器函数

        iterable = True

    except TypeError as e:

        iterable = False

    # Undo our custom logging setup.

    restore_defaults()

    # Set up the root logger with a custom handler that logs stdout for INFO

    # and DEBUG and stderr for WARNING, ERROR, CRITICAL.

    root_logger = logging.getLogger() #logging主要是python中用于处理日志的模块,这里是返回一个logger实例，由于没有指定name，所以是root logger

    if args.verbose_logging:

        formatter = logging.Formatter(fmt="%(asctime)s %(name)s %(levelname)s "

                                          "%(message)s")

        handler = CustomStreamHandler(formatter=formatter)

    else:

        if args.timestamp:

            prefix = '%(asctime)s '

        else:

            prefix = '' #这里为空

        formatter = CustomFormatter(prefix=prefix, only_from='pylearn2')

        handler = CustomStreamHandler(formatter=formatter)

    root_logger.addHandler(handler) #给root_lgger添加handler来帮助处理日志

    # Set the root logger level.

    if args.debug:

        root_logger.setLevel(logging.DEBUG)

    else:

        root_logger.setLevel(logging.INFO) #给root_logger设置级别，为INFO级别，因为每个日志消息都会关联一个级别

    if iterable: #enumerate()为对一个list或者array既要遍历索引又要遍历元素时使用

        for number, subobj in enumerate(iter(train_obj)):#train_obj里面装的是ymal文件内容，类似字典

            # Publish a variable indicating the training phase.

            phase_variable = 'PYLEARN2_TRAIN_PHASE'

            phase_value = 'phase%d' % (number + 1)

            os.environ[phase_variable] = phase_value

            os.putenv(phase_variable, phase_value)

            # Execute this training phase.

            subobj.main_loop()

            # Clean up, in case there's a lot of memory used that's

            # necessary for the next phase.

            del subobj

            gc.collect()

    else:

        train_obj.main_loop() #因为train_obj中已经包含了数据，模型，算法，所以调用main_loop()后表示采用对应算法用对应数据在对应的模型上训练

                              #直到满足迭代终止条件

　　首先是由train_obj.main_loop()函数将data, model, algorithm联系起来的。从名字train_obj可以看出它是一个某个类的对象，猜测应该是Pylearn2下的Train类对象，因为在库Pylearn2的子目录下有个model为train.py，该文件有个Train类，并且这个Train类有一个方法：main_loop()。看来一切符合猜测，那么是否真是的呢？

　　首先来看看train_obj从哪里来的(因为main_loop()是由train_obj来调用的)。由上面的程序可知：train_obj = serial.load_train_file(args.config), 需要跟踪serial, 找到serial.load_train_file()的源代码，最后一句为：return yaml_parse.load_path(args.config). 继续跟踪发现load_path()函数里面调用了load()函数，而里面最调用的是yaml.load()函数，由源码中的注释可知它是将.yaml配置文件转换成一个graph, 而这个graph应该就是一个Train对象...

　　好吧，到了该看main_loop()的内容了：

    def main_loop(self):

        """

        Repeatedly runs an epoch of the training algorithm, runs any

        epoch-level callbacks, and saves the model.

        """

        if self.algorithm is None:

            self.model.monitor = Monitor.get_monitor(self.model)

            self.setup_extensions()

            self.run_callbacks_and_monitoring()

            while True:

                rval = self.model.train_all(dataset=self.dataset)

                if rval is not None:

                    raise ValueError("Model.train_all should not return anything. Use Model.continue_learning to control whether learning continues.")

                self.model.monitor.report_epoch()

                if self.save_freq > 0 and self.model.monitor.epochs_seen % self.save_freq == 0:

                    self.save()

                continue_learning = self.model.continue_learning()

                assert continue_learning in [True, False, 0, 1]

                if not continue_learning:

                    break

        else:

            self.algorithm.setup(model=self.model, dataset=self.dataset) #这一句将data,model, dataset联系起来了

            self.setup_extensions() #和.yaml文件中的extensions项联系起来了

            if not hasattr(self.model, 'monitor'):

                # TODO: is this really necessary? I just put this error here

                # to prevent an AttributeError later, but I think we could

                # rewrite to avoid the AttributeError

                raise RuntimeError("The algorithm is responsible for setting"

                        " up the Monitor, but failed to.")

            if len(self.model.monitor._datasets)>0:

                # This monitoring channel keeps track of a shared variable,

                # which does not need inputs nor data.

                self.model.monitor.add_channel(name="monitor_seconds_per_epoch",

                                               ipt=None,

                                               val=self.monitor_time,

                                               data_specs=(NullSpace(), ''),

                                               dataset=self.model.monitor._datasets[0])

            self.run_callbacks_and_monitoring()

            while True: #循环中，直到满足终止条件

                with log_timing(log, None, final_msg='Time this epoch:',

                                callbacks=[self.monitor_time.set_value]):

                    rval = self.algorithm.train(dataset=self.dataset) #算法训练的核心函数

                if rval is not None:

                    raise ValueError("TrainingAlgorithm.train should not return anything. Use TrainingAlgorithm.continue_learning to control whether learning continues.")

                self.model.monitor.report_epoch()

                self.run_callbacks_and_monitoring()

                if self.save_freq > 0 and self.model.monitor._epochs_seen % self.save_freq == 0:

                    self.save()

                continue_learning =  self.algorithm.continue_learning(self.model) #终止条件测试

                assert continue_learning in [True, False, 0, 1]

                if not continue_learning:

                    break

        self.model.monitor.training_succeeded = True

        if self.save_freq > 0:

            self.save()