用CNTK搞深度学习（二）训练基于RNN的自然语言模型(languagemodel)

# Parameters can be overwritten on the command line
# for example: cntk cOnfigFile=myConfigFile RootDir=../.. 
# For running from Visual Studio add
# currentDirectory=$(SolutionDir)/ 
RootDir = ".."

ConfigDir = "$RootDir$/Config"
DataDir = "$RootDir$/Data"
OutputDir = "$RootDir$/Output"
ModelDir = "$OutputDir$/Models"

# deviceId=-1 for CPU, >=0 for GPU devices, "auto" chooses the best GPU, or CPU if no usable GPU is available
deviceId = "-1"

command = writeWordAndClassInfo:train
#command = write

precision = "float"
traceLevel = 1
modelPath = "$ModelDir$/rnn.dnn"

# uncomment the following line to write logs to a file
stderr=$OutputDir$/rnnOutput

type = double
numCPUThreads = 4

confVocabSize = 3000
confClassSize = 50

#trainFile = "ptb.train.txt"
trainFile = "review_tokens_split_first5w_lines.txt"
#validFile = "ptb.valid.txt"
testFile = "review_tokens_split_first10_lines.txt"

writeWordAndClassInfo = [
    action = "writeWordAndClass"
    inputFile = "$DataDir$/$trainFile$"
    outputVocabFile = "$ModelDir$/vocab.txt"
    outputWord2Cls = "$ModelDir$/word2cls.txt"
    outputCls2Index = "$ModelDir$/cls2idx.txt"
    vocabSize = "$confVocabSize$"
    nbrClass = "$confClassSize$"
    cutoff = 1
    printValues = true
]

#######################################
#  TRAINING CONFIG                    #
#######################################

train = [
    action = "train"
    minibatchSize = 10
    traceLevel = 1
    epochSize = 0
    recurrentLayer = 1
    defaultHiddenActivity = 0.1
    useValidation = true
    rnnType = "CLASSLM"

     # uncomment below and comment SimpleNetworkBuilder section to use NDL to train RNN LM
     NDLNetworkBuilder=[
        networkDescription="D:\tools\Deep Learning\CNTK-2016-02-08-Windows-64bit-CPU-Only\cntk\Examples\Text\PennTreebank\AdditionalFiles\RNNLM\rnnlm.ndl"
     ]
  

    SGD = [
        learningRatesPerSample = 0.1
        momentumPerMB = 0
        gradientClippingWithTruncation = true
        clippingThresholdPerSample = 15.0
        maxEpochs = 6
        unroll = false
        numMBsToShowResult = 100
        gradUpdateType = "none"
        loadBestModel = true

        # settings for Auto Adjust Learning Rate
        AutoAdjust = [
            autoAdjustLR = "adjustAfterEpoch"
            reduceLearnRateIfImproveLessThan = 0.001
            continueReduce = false
            increaseLearnRateIfImproveMoreThan = 1000000000
            learnRateDecreaseFactor = 0.5
            learnRateIncreaseFactor = 1.382
            numMiniBatch4LRSearch = 100
            numPrevLearnRates = 5
            numBestSearchEpoch = 1
        ]

        dropoutRate = 0.0
    ]

    reader = [
        readerType = "LMSequenceReader"
        randomize = "none"
        nbruttsineachrecurrentiter = 16

        # word class info
        wordclass = "$ModelDir$/vocab.txt"

        # if writerType is set, we will cache to a binary file
        # if the binary file exists, we will use it instead of parsing this file
        # writerType=BinaryReader

        # write definition
        wfile = "$OutputDir$/sequenceSentence.bin"
        
        # wsize - inital size of the file in MB
        # if calculated size would be bigger, that is used instead
        wsize = 256

        # wrecords - number of records we should allocate space for in the file
        # files cannot be expanded, so this should be large enough. If known modify this element in config before creating file
        wrecords = 1000
        
        # windowSize - number of records we should include in BinaryWriter window
        windowSize = "$confVocabSize$"

        file = "$DataDir$/$trainFile$"

        # additional features sections
        # for now store as expanded category data (including label in)
        features = [
            # sentence has no features, so need to set dimension to zero
            dim = 0
            # write definition
            sectionType = "data"
        ]
      
        # sequence break table, list indexes into sequence records, so we know when a sequence starts/stops
        sequence = [
            dim = 1
            wrecords = 2
            # write definition
            sectionType = "data"
        ]
        
        #labels sections
        labelIn = [
            dim = 1
            labelType = "Category"
            beginSequence = ""
            endSequence = ""

            # vocabulary size
            labelDim = "$confVocabSize$"
            labelMappingFile = "$OutputDir$/sentenceLabels.txt"
            
            # Write definition
            # sizeof(unsigned) which is the label index type
            elementSize = 4
            sectionType = "labels"
            mapping = [
                # redefine number of records for this section, since we don‘t need to save it for each data record
                wrecords = 11                
                # variable size so use an average string size
                elementSize = 10
                sectionType = "labelMapping"
            ]
            
            category = [
                dim = 11
                # elementSize = sizeof(ElemType) is default
                sectionType = "categoryLabels"
            ]
        ]
        
        # labels sections
        labels = [
            dim = 1
            labelType = "NextWord"
            beginSequence = "O"
            endSequence = "O"

            # vocabulary size
            labelDim = "$confVocabSize$"
            labelMappingFile = "$OutputDir$/sentenceLabels.out.txt"
            
            # Write definition 
            # sizeof(unsigned) which is the label index type
            elementSize = 4
            sectionType = "labels"
            mapping = [
                # redefine number of records for this section, since we don‘t need to save it for each data record
                wrecords = 3
                # variable size so use an average string size
                elementSize = 10
                sectionType = "labelMapping"
            ]
            
            category = [
                dim = 3
                # elementSize = sizeof(ElemType) is default
                sectionType = categoryLabels
            ]
        ]
    ] 
]



write = [
    action = "write"

    outputPath = "$OutputDir$/Write"
    #outputPath = "-"                    # "-" will write to stdout; useful for debugging
    outputNodeNames = "Out,WFeat2Hid,WHid2Hid,WHid2Word" # when processing one sentence per minibatch, this is the sentence posterior
    #format = [
        #sequencePrologue = "log P(W)="    # (using this to demonstrate some formatting strings)
        #type = "real"
    #]

    minibatchSize = 1              # choose this to be big enough for the longest sentence
    # need to be small since models are updated for each minibatch
    traceLevel = 1
    epochSize = 0

    reader = [
        # reader to use
        readerType = "LMSequenceReader"
        randomize = "none"              # BUGBUG: This is ignored.
        nbruttsineachrecurrentiter = 1  # one sentence per minibatch
        cacheBlockSize = 1              # workaround to disable randomization

        # word class info
        wordclass = "$ModelDir$/vocab.txt"

        # if writerType is set, we will cache to a binary file
        # if the binary file exists, we will use it instead of parsing this file
        # writerType = "BinaryReader"

        # write definition
        wfile = "$OutputDir$/sequenceSentence.bin"
        # wsize - inital size of the file in MB
        # if calculated size would be bigger, that is used instead
        wsize = 256

        # wrecords - number of records we should allocate space for in the file
        # files cannot be expanded, so this should be large enough. If known modify this element in config before creating file
        wrecords = 1000
        
        # windowSize - number of records we should include in BinaryWriter window
        windowSize = "$confVocabSize$"

        file = "$DataDir$/$testFile$"

        # additional features sections
        # for now store as expanded category data (including label in)
        features = [
            # sentence has no features, so need to set dimension to zero
            dim = 0
            # write definition
            sectionType = "data"
        ]
        
        #labels sections
        labelIn = [
            dim = 1

            # vocabulary size
            labelDim = "$confVocabSize$"
            labelMappingFile = "$OutputDir$/sentenceLabels.txt"
            
            labelType = "Category"
            beginSequence = ""
            endSequence = ""

            # Write definition
            # sizeof(unsigned) which is the label index type
            elementSize = 4
            sectionType = "labels"
            
            mapping = [
                # redefine number of records for this section, since we don‘t need to save it for each data record
                wrecords = 11
                # variable size so use an average string size
                elementSize = 10
                sectionType = "labelMapping"
            ]
            
            category = [
                dim = 11
                # elementSize = sizeof(ElemType) is default
                sectionType = "categoryLabels"
            ]
        ]
        
        #labels sections
        labels = [
            dim = 1
            labelType = "NextWord"
            beginSequence = "O"
            endSequence = "O"

            # vocabulary size
            labelDim = "$confVocabSize$"

            labelMappingFile = "$OutputDir$/sentenceLabels.out.txt"
            # Write definition
            # sizeof(unsigned) which is the label index type
            elementSize = 4
            sectionType = "labels"
            
            mapping = [
                # redefine number of records for this section, since we don‘t need to save it for each data record
                wrecords = 3
                # variable size so use an average string size
                elementSize = 10
                sectionType = "labelMapping"
            ]
            
            category = [
                dim = 3
                # elementSize = sizeof(ElemType) is default
                sectionType = "categoryLabels"
            ]
        ]
    ]
]    

run=ndlCreateNetwork

ndlCreateNetwork=[
    # vocabulary size
    featDim=3000
    # vocabulary size
    labelDim=3000
    # hidden layer size
    hiddenDim=200
    # number of classes
    nbrClass=50
    
    initScale=6
    
    features=SparseInput(featDim, tag="feature")
    
    # labels in classbasedCrossEntropy is dense and contain 4 values for each sample
    labels=Input(4, tag="label")

    # define network
    WFeat2Hid=Parameter(hiddenDim, featDim, init="uniform", initValueScale=initScale)
    WHid2Hid=Parameter(hiddenDim, hiddenDim, init="uniform", initValueScale=initScale)

    # WHid2Word is special that it is hiddenSize X labelSize
    WHid2Word=Parameter( hiddenDim,labelDim,  init="uniform", initValueScale=initScale)
     WHid2Class=Parameter(nbrClass, hiddenDim, init="uniform", initValueScale=initScale)
   
    PastHid = Delay(hiddenDim, HidAfterSig, delayTime=1, needGradient=true)    
    HidFromHeat = Times(WFeat2Hid, features)
    HidFromRecur = Times(WHid2Hid, PastHid)
    HidBeforeSig = Plus(HidFromHeat, HidFromRecur)
    HidAfterSig = Sigmoid(HidBeforeSig)
    
    Out = TransposeTimes(WHid2Word, HidAfterSig)  #word part
    
    ClassProbBeforeSoftmax=Times(WHid2Class, HidAfterSig)
    
    cr = ClassBasedCrossEntropyWithSoftmax(labels, HidAfterSig, WHid2Word, ClassProbBeforeSoftmax, tag="criterion")
    EvalNodes=(Cr)
    OutputNodes=(Cr)
]

SimpleNetworkBuilder=[
        trainingCriterion=classcrossentropywithsoftmax
        evalCriterion=classcrossentropywithsoftmax
        nodeType=Sigmoid
        initValueScale=6.0
        layerSizes=10000:200:10000
        addPrior=false
        addDropoutNodes=false
        applyMeanVarNorm=false
        uniformInit=true;

        # these are for the class information for class-based language modeling
        vocabSize=10000
        nbrClass=50
    ]