diff --git a/generator/knn.py b/generator/knn.py index 54860fd..f61e75e 100644 --- a/generator/knn.py +++ b/generator/knn.py @@ -3,25 +3,28 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import numpy as np -import os +from __future__ import print_function import operator from math import sqrt -import random -from ast import literal_eval from copy import deepcopy +from future.utils import iteritems from loader.DataReader import * from loader.GentScorer import * -from ConfigParser import SafeConfigParser +try: + + from ConfigParser import SafeConfigParser +except ImportError: + + from configparser import SafeConfigParser class KNN(object): def __init__(self,config=None,opts=None): # not enough info to execute if config==None and opts==None: - print "Please specify command option or config file ..." + print("Please specify command option or config file ...") return # config parser parser = SafeConfigParser() @@ -63,7 +66,7 @@ def testKNN(self): ######## test KNN generator on test set ######### if self.debug: - print 'start KNN generation ...' + print('start KNN generation ...') # container parallel_corpus, hdc_corpus = [], [] @@ -81,7 +84,7 @@ def testKNN(self): sents,dact,bases = sents[0],dact[0],bases[0] # score DA similarity between testing example and train+valid set template_ranks = [] - for da_t,sents_t in da2sents.iteritems(): + for da_t, sents_t in iteritems(da2sents): a_t,sv_t = [set(x) for x in da_t] score =float(len(a_t.intersection(set(a)))+\ len(sv_t.intersection(set(sv))))/\ @@ -96,9 +99,9 @@ def testKNN(self): # for slot error rate scoring felements = [self.reader.cardinality[x+self.reader.dfs[1]]\ for x in sv] - # print results - print dact - print 'Sim\tTSER\tASER\tGen' + # print(results) + print(dact) + print('Sim\tTSER\tASER\tGen') for i in range(len(gens)): # score slot error rate cnt, total, caty = self.gentscorer.scoreERR(a,felements, @@ -108,8 +111,8 @@ def testKNN(self): gencnts[0] += cnt gencnts[1] += total gencnts[2] += caty - print '%.4f\t%d\t%d\t%s' % (score,total,caty,gens[i]) - print '\n' + print('%.4f\t%d\t%d\t%s' % (score,total,caty,gens[i])) + print('\n') # compute gold standard slot error rate for sent in sents: @@ -127,16 +130,16 @@ def testKNN(self): bleuModel = self.gentscorer.scoreBLEU(parallel_corpus) bleuHDC = self.gentscorer.scoreBLEU(hdc_corpus) - print '##############################################' - print 'BLEU SCORE & SLOT ERROR on GENERATED SENTENCES' - print '##############################################' - print 'Metric :\tBLEU\tT.ERR\tA.ERR' - print 'HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0) - print 'Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, - 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0]) - print '----------------------------------------------' - print 'This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, - 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0]) + print('##############################################') + print('BLEU SCORE & SLOT ERROR on GENERATED SENTENCES') + print('##############################################') + print('Metric :\tBLEU\tT.ERR\tA.ERR') + print('HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0)) + print('Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, + 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0])) + print('----------------------------------------------') + print('This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, + 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0])) def setupSideOperators(self): # initialise data reader diff --git a/generator/net.py b/generator/net.py index ff1ec14..2f9eef7 100644 --- a/generator/net.py +++ b/generator/net.py @@ -3,27 +3,25 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import theano -import numpy as np +from __future__ import print_function + import os import operator from math import log, log10, exp, pow -import sys -import random import time -import itertools import pickle as pk -from ast import literal_eval - -from theano import tensor as T -from collections import OrderedDict from nn.NNGenerator import * from loader.DataReader import * from loader.GentScorer import * -from ConfigParser import SafeConfigParser +try: + + from ConfigParser import SafeConfigParser +except ImportError: + + from configparser import SafeConfigParser # theano debugging flags """ @@ -57,7 +55,7 @@ class Model(object): def __init__(self,config=None,opts=None): # not enough info to execute if config==None and opts==None: - print "Please specify command option or config file ..." + print("Please specify command option or config file ...") return # config parser parser = SafeConfigParser() @@ -75,7 +73,7 @@ def __init__(self,config=None,opts=None): def initNet(self,config,opts=None): - print '\n\ninit net from scrach ... ' + print('\n\ninit net from scrach ... ') # config parser parser = SafeConfigParser() @@ -84,7 +82,7 @@ def initNet(self,config,opts=None): # setting learning hyperparameters self.debug = parser.getboolean('learn','debug') if self.debug: - print 'loading settings from config file ...' + print('loading settings from config file ...') self.seed = parser.getint( 'learn','random_seed') self.lr_divide = parser.getint( 'learn','lr_divide') self.lr = parser.getfloat('learn','lr') @@ -99,7 +97,7 @@ def initNet(self,config,opts=None): self.batch = parser.getint('train_mode','batch') # setting file paths if self.debug: - print 'loading file path from config file ...' + print('loading file path from config file ...') self.wvecfile = parser.get('data','wvec') self.trainfile = parser.get('data','train') self.validfile = parser.get('data','valid') @@ -136,13 +134,12 @@ def initModel(self): #################### Model Initialisation ####################### ################################################################# if self.debug: - print 'setting network structures using theano variables ...' + print('setting network structures using theano variables ...') ########################################################### ############## Setting Recurrent Generator ################ ########################################################### if self.debug: - print '\tsetting recurrent generator, type: %s ...' % \ - self.gentype + print('\tsetting recurrent generator, type: %s ...' % self.gentype) self.model = NNGenerator(self.gentype, self.reader.vocab, self.beamwidth, self.overgen, self.di, self.dh, self.batch, self.reader.dfs, @@ -153,9 +150,8 @@ def initModel(self): self.model.setWordVec(self.reader.readVecFile( self.wvecfile,self.reader.vocab)) if self.debug: - print '\t\tnumber of parameters : %8d' % \ - self.model.numOfParams() - print '\tthis may take up to several minutes ...' + print('\t\tnumber of parameters : %8d' % self.model.numOfParams()) + print('\tthis may take up to several minutes ...') ################################################################# ####################### Training ################################ @@ -169,7 +165,7 @@ def trainNet(self): def trainNetML(self): ######## training RNN generator with early stopping ######### if self.debug: - print 'start network training ...' + print('start network training ...') epoch = 0 lr_divide = 0 llr_divide= -1 @@ -196,14 +192,13 @@ def trainNetML(self): num_sent+=cutoff_b # log message if self.debug and num_sent%100==0: - print 'Finishing %8d sent in epoch %3d\r' % \ - (num_sent,epoch), + print('Finishing %8d sent in epoch %3d\r' % (num_sent,epoch), end="") sys.stdout.flush() # log message sec = (time.time()-tic)/60.0 if self.debug: - print 'Epoch %3d, Alpha %.6f, TRAIN entropy:%.2f, Time:%.2f mins,' %\ - (epoch, self.lr, -train_logp/log10(2)/wcn, sec), + print('Epoch %3d, Alpha %.6f, TRAIN entropy:%.2f, Time:%.2f mins,' %\ + (epoch, self.lr, -train_logp/log10(2)/wcn, sec), end="") sys.stdout.flush() # validation phase @@ -221,7 +216,7 @@ def trainNetML(self): wcn += np.sum(cutoff_f-1) # log message if self.debug: - print 'VALID entropy:%.2f'%-(self.valid_logp/log10(2)/wcn) + print('VALID entropy:%.2f'%-(self.valid_logp/log10(2)/wcn)) # decide to throw/keep weights if self.valid_logp < self.llogp: @@ -239,7 +234,7 @@ def trainNetML(self): lr_divide += 1 else: self.saveNet() - print 'Training completed.' + print('Training completed.') break # set last epoch objective value self.llogp = self.valid_logp @@ -248,7 +243,7 @@ def trainNetML(self): def trainNetDT(self): # start if self.debug: - print 'start network training with expected objective ...' + print('start network training with expected objective ...') # examples train_examples = self.reader.readall(mode='train') @@ -327,12 +322,12 @@ def trainNetDT(self): train_obj += xObj num_sent+=1 if self.debug and num_sent%1==0: - print 'Finishing %8d sent in epoch %3d\r' % \ - (num_sent,epoch), + print('Finishing %8d sent in epoch %3d\r' % \ + (num_sent,epoch), end="") sys.stdout.flush() sec = (time.time()-tic)/60.0 if self.debug: - print 'Epoch %2d, Alpha %.4f, TRAIN Obj:%.4f, Expected BLEU:%.4f, Expected ERR:%.4f, Time:%.2f mins,' % (epoch, self.lr, train_obj/float(num_sent), train_bleu/float(num_sent), train_err/float(num_sent), sec), + print('Epoch %2d, Alpha %.4f, TRAIN Obj:%.4f, Expected BLEU:%.4f, Expected ERR:%.4f, Time:%.2f mins,' % (epoch, self.lr, train_obj/float(num_sent), train_bleu/float(num_sent), train_err/float(num_sent), sec), end="") sys.stdout.flush() # validation phase @@ -389,7 +384,7 @@ def trainNetDT(self): num_sent +=1 if self.debug: - print 'VALID Obj:%.3f'% (self.valid_obj/float(num_sent)) + print('VALID Obj:%.3f'% (self.valid_obj/float(num_sent))) # decide to throw/keep weights if self.valid_obj > self.lobj: # throw weight @@ -403,7 +398,7 @@ def trainNetDT(self): lr_divide += 1 else: self.saveNet() - print 'Training completed.' + print('Training completed.') break if self.valid_obj < self.lobj: @@ -419,7 +414,7 @@ def trainNetDT(self): def testNet(self): ######## test RNN generator on test set ######### if self.debug: - print 'start network testing ...' + print('start network testing ...') self.model.loadConverseParams() # container @@ -455,9 +450,9 @@ def testNet(self): gens[i] = (penalty,self.reader.lexicalise(gen,dact)) # get the top-k for evaluation gens = sorted(gens,key=operator.itemgetter(0))[:self.topk] - # print results - print dact - print 'Penalty\tTSER\tASER\tGen' + # print(results) + print(dact) + print('Penalty\tTSER\tASER\tGen') for penalty, gen in gens: # score slot error rate cnt, total, caty = self.gentscorer.scoreERR(a,felements, @@ -466,8 +461,8 @@ def testNet(self): gencnts[0] += cnt gencnts[1] += total gencnts[2] += caty - print '%.4f\t%d\t%d\t%s' % (penalty,total,caty,gen) - print '\n' + print('%.4f\t%d\t%d\t%s' % (penalty,total,caty,gen)) + print('\n') # compute gold standard slot error rate for sent in sents: @@ -485,16 +480,16 @@ def testNet(self): bleuModel = self.gentscorer.scoreBLEU(parallel_corpus) bleuHDC = self.gentscorer.scoreBLEU(hdc_corpus) - print '##############################################' - print 'BLEU SCORE & SLOT ERROR on GENERATED SENTENCES' - print '##############################################' - print 'Metric :\tBLEU\tT.ERR\tA.ERR' - print 'HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0) - print 'Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, - 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0]) - print '----------------------------------------------' - print 'This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, - 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0]) + print('##############################################') + print('BLEU SCORE & SLOT ERROR on GENERATED SENTENCES') + print('##############################################') + print('Metric :\tBLEU\tT.ERR\tA.ERR') + print('HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0)) + print('Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, + 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0])) + print('----------------------------------------------') + print('This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, + 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0])) ################################################################# @@ -510,21 +505,22 @@ def updateNumpyParams(self): def saveNet(self): if self.debug: - print 'saving net to file ... ' + print('saving net to file ... ') self.updateNumpyParams() + locals_ = locals() bundle={ - 'learn' :dict( [(name,eval(name)) for name in self.learn_vars] ), - 'data' :dict( [(name,eval(name)) for name in self.data_vars] ), - 'gen' :dict( [(name,eval(name)) for name in self.gen_vars] ), - 'model' :dict( [(name,eval(name)) for name in self.model_vars] ), - 'mode' :dict( [(name,eval(name)) for name in self.mode_vars] ), - 'params':dict( [(name,eval(name)) for name in self.params_vars] ) + 'learn' :dict( [(name,eval(name, globals(), locals_)) for name in self.learn_vars] ), + 'data' :dict( [(name,eval(name, globals(), locals_)) for name in self.data_vars] ), + 'gen' :dict( [(name,eval(name, globals(), locals_)) for name in self.gen_vars] ), + 'model' :dict( [(name,eval(name, globals(), locals_)) for name in self.model_vars] ), + 'mode' :dict( [(name,eval(name, globals(), locals_)) for name in self.mode_vars] ), + 'params':dict( [(name,eval(name, globals(), locals_)) for name in self.params_vars] ) } pk.dump(bundle, open(self.modelfile, 'wb')) def loadNet(self,parser,mode): - print '\n\nloading net from file %s ... ' % self.modelfile + print('\n\nloading net from file %s ... ' % self.modelfile) bundle = pk.load(open(self.modelfile, 'rb')) # load learning variables from model # if adaptation, load from config file diff --git a/generator/ngram.py b/generator/ngram.py index eb938d8..eacef37 100644 --- a/generator/ngram.py +++ b/generator/ngram.py @@ -3,27 +3,29 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import numpy as np -import os -import operator +from __future__ import print_function from math import sqrt -import random -from ast import literal_eval -from copy import deepcopy from loader.DataReader import * from loader.GentScorer import * from nn.ngmodel import * -from ConfigParser import SafeConfigParser +from future.utils import iteritems + +try: + + from ConfigParser import SafeConfigParser +except ImportError: + + from configparser import SafeConfigParser class Ngram(object): def __init__(self,config=None,opts=None): # not enough info to execute if config==None and opts==None: - print "Please specify command option or config file ..." + print("Please specify command option or config file ...") return # config parser parser = SafeConfigParser() @@ -59,30 +61,30 @@ def testNgram(self): ######## train ngram generator by grouping ######## if self.debug: - print 'start ngram training ...' + print('start ngram training ...') da2sents = {} templates = self.reader.readall(mode='train')+\ self.reader.readall(mode='valid') for a,sv,s,v,sents,dact,base in templates: key = (tuple(a),tuple(sv)) - if da2sents.has_key(key): + if key in da2sents: da2sents[key].extend(sents) da2sents[key] = list(set(da2sents[key])) else: da2sents[key] = sents # accumulate texts for training class-based LM cls2texts = {} - for key,sents in da2sents.iteritems(): + for key,sents in iteritems(da2sents): a,sv = key identifier = (a,sv[:self.rho]) if len(sv)>self.rho else (a,sv) - if cls2texts.has_key(identifier): + if identifier in cls2texts: cls2texts[identifier].extend(sents) else: cls2texts[identifier] = sents # train class based ngram models cls2model = {} - for key, sents in cls2texts.iteritems(): + for key, sents in iteritems(cls2texts): model = NGModel(self.reader.vocab,self.topk,self.overgen, self.beamwidth,n=self.N,rho=self.rho) model.train(sents) @@ -90,7 +92,7 @@ def testNgram(self): ######## test ngram generator on test set ######### if self.debug: - print 'start ngram generation ...' + print('start ngram generation ...') # container parallel_corpus, hdc_corpus = [], [] @@ -108,7 +110,7 @@ def testNgram(self): sents,dact,bases = sents[0],dact[0],bases[0] # score DA similarity between testing example and class LMs model_ranks = [] - for da_t,model in cls2model.iteritems(): + for da_t, model in iteritems(cls2model): a_t,sv_t = [set(x) for x in da_t] # cosine similarity score =float(len(a_t.intersection(set(a)))+\ @@ -140,9 +142,9 @@ def testNgram(self): gens[i] = (penalty,self.reader.lexicalise(gen,dact)) # get the top-k for evaluation gens = sorted(gens,key=operator.itemgetter(0))[:self.topk] - # print results - print dact - print 'Penalty\tTSER\tASER\tGen' + # print(results) + print(dact) + print('Penalty\tTSER\tASER\tGen') for penalty, gen in gens: # score slot error rate cnt, total, caty = self.gentscorer.scoreERR(a,felements, @@ -151,8 +153,8 @@ def testNgram(self): gencnts[0] += cnt gencnts[1] += total gencnts[2] += caty - print '%.4f\t%d\t%d\t%s' % (penalty,total,caty,gen) - print '\n' + print('%.4f\t%d\t%d\t%s' % (penalty,total,caty,gen)) + print('\n') # compute gold standard slot error rate for sent in sents: # score slot error rate @@ -168,16 +170,16 @@ def testNgram(self): bleuModel = self.gentscorer.scoreBLEU(parallel_corpus) bleuHDC = self.gentscorer.scoreBLEU(hdc_corpus) - print '##############################################' - print 'BLEU SCORE & SLOT ERROR on GENERATED SENTENCES' - print '##############################################' - print 'Metric :\tBLEU\tT.ERR\tA.ERR' - print 'HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0) - print 'Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, - 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0]) - print '----------------------------------------------' - print 'This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, - 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0]) + print('##############################################') + print('BLEU SCORE & SLOT ERROR on GENERATED SENTENCES') + print('##############################################') + print('Metric :\tBLEU\tT.ERR\tA.ERR') + print('HDC :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuHDC,0.0,0.0)) + print('Ref :\t%.4f\t%2.2f%%\t%2.2f%%'% (1.0, + 100*refcnts[1]/refcnts[0],100*refcnts[2]/refcnts[0])) + print('----------------------------------------------') + print('This Model :\t%.4f\t%2.2f%%\t%2.2f%%'% (bleuModel, + 100*gencnts[1]/gencnts[0],100*gencnts[2]/gencnts[0])) def setupSideOperators(self): # initialise data reader diff --git a/loader/DataLexicaliser.py b/loader/DataLexicaliser.py index 185f1ef..13e2adf 100644 --- a/loader/DataLexicaliser.py +++ b/loader/DataLexicaliser.py @@ -11,17 +11,18 @@ import nltk import json +file = open class DataLexicaliser(object): def __init__(self): fin = file('resource/special_values.txt') - self.special_values= json.load(fin).keys() + ['?'] + self.special_values= list(json.load(fin).keys()) + ['?'] fin.close() fin = file('resource/special_slots.txt') self.special_slots = json.load(fin) fin.close() - + def delexicalise(self,sent,jssv): raise NotImplementedError('method delexicalise() hasn\'t been implemented') def lexicalise(self,sent,jssv): @@ -37,43 +38,43 @@ def delexicalise(self,sent,jssv): # no slot values return directly if len(jssv)==1 and jssv[0][1]==None: return sent - for slot,value in sorted(jssv,key=lambda x:len(x[-1]),reverse=True): - if value in self.special_values : continue # special values, skip + for slot,value in sorted(jssv,key=lambda x:len(x[-1]),reverse=True): + if value in self.special_values : continue # special values, skip # taking care of all possible permutations of multiple values vs = value.replace(' or ',' and ').split(' and ') permutations = [' and '.join(x) for x in itertools.permutations(vs)]+\ [' or '.join(x) for x in itertools.permutations(vs)] - + # try to match for each possible permutation isMatched = False for p in permutations: - if p in sent : # exact match , ends + if p in sent : # exact match , ends sent = (' '+sent+' ').replace(\ ' '+p+' ',' SLOT_'+slot.upper()+' ',1)[1:-1] isMatched = True break - if not isMatched: + if not isMatched: pass #raise ValueError('value "'+value+'" cannot be delexicalised!') - + return sent def lexicalise(self,sent,jssv): # no slot values return directly if len(jssv)==1 and jssv[0][1]==None: return sent - - # replace values + + # replace values for slot,value in sorted(jssv,key=lambda x:len(x[0]),reverse=True): - if value in self.special_values : continue # special values, skip - if 'SLOT_'+slot.upper() not in sent : + if value in self.special_values : continue # special values, skip + if 'SLOT_'+slot.upper() not in sent : pass #raise ValueError('slot "SLOT_'+slot.upper()+'" does not exist !') - else: + else: sent=(' '+sent+' ').replace(' SLOT_'+slot.upper()+' ',' '+value+' ',1)[1:-1] sent = (' '+sent+' ').replace(' SLOT_TYPE ',' '+self.typetoken+' ')[1:-1] - return sent + return sent #if __name__ == '__main__': diff --git a/loader/DataReader.py b/loader/DataReader.py index 0e1a1c4..f02040a 100644 --- a/loader/DataReader.py +++ b/loader/DataReader.py @@ -3,20 +3,18 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import os import re -import sys -import json -import math -import operator import random -import itertools import numpy as np -from FeatParser import * -from DataLexicaliser import * +from builtins import input as raw_input +from future.utils import iteritems + +from .FeatParser import * +from .DataLexicaliser import * from utils.nlp import * +file = open class DataReader(object): def __init__(self, seed, domain, obj, @@ -113,8 +111,8 @@ def read(self,mode='train',batch=1): if len(xvec[j])>max_leng[j]: max_leng[j] = len(xvec[j]) else: # TODO:DT training, 1 da/multiple sents per example - print a,sv - print sent + print(a, sv) + print(sent) raw_input() # padding to have the same sent length lengs = [[],[],[],[],[]] @@ -175,14 +173,14 @@ def _setCardinality(self): self.dfs[i+1] = self.dfs[i] + self.dfs[i+1] def _printStats(self): - print '===============' - print 'Data statistics' - print '===============' - print 'Train: %d' % len(self.data['train'] ) - print 'Valid: %d' % len(self.data['valid'] ) - print 'Test : %d' % len(self.data['test'] ) - print 'Feat : %d' % len(self.cardinality) - print '===============' + print('===============') + print('Data statistics') + print('===============') + print('Train: %d' % len(self.data['train'] )) + print('Valid: %d' % len(self.data['valid'] )) + print('Test : %d' % len(self.data['test'] )) + print('Feat : %d' % len(self.cardinality)) + print('===============') def _testDelexicalisation(self): for data in self.data['train']+self.data['valid']+self.data['test']: @@ -227,7 +225,7 @@ def _loadData(self,filename,group=True,multiref=False): # grouping data points according to unique DAs a2ref = {} for feat,dact,sent,base in container: - if a2ref.has_key(tuple(feat)): + if tuple(feat) in a2ref: a2ref[ tuple(feat)][0].append(dact) a2ref[ tuple(feat)][1].append(sent) a2ref[ tuple(feat)][2].append(base) @@ -273,7 +271,7 @@ def _loadTokenMap(self,mapfile='resource/detect.pair'): def tokenMap2Indexes(self): maxleng = 0 idxmap = [[] for x in range(len(self.vocab))] - for k,v in self.feat2token.iteritems(): + for k,v in iteritems(self.feat2token): try: idxmap[self.vocab.index(v)].append(self.cardinality.index(k)-self.dfs[1]) if len(idxmap[self.vocab.index(v)])>maxleng: diff --git a/loader/FeatParser.py b/loader/FeatParser.py index 5d06a80..254a7a3 100644 --- a/loader/FeatParser.py +++ b/loader/FeatParser.py @@ -3,9 +3,12 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import sys -import os +from __future__ import print_function + import json +from future.utils import iteritems + +file = open class DialogActParser(object): @@ -36,10 +39,10 @@ def parse(self,dact,keepValues=False): else: # both slot and value exist s,v = [x.strip('\'\"') for x in slt2val.split('=')] s = s.replace('_','').replace(' ','') - for key,vals in self.special_values.iteritems(): + for key, vals in iteritems(self.special_values): if v in vals: # unify the special values v = key - if not self.special_values.has_key(v) and\ + if not v in self.special_values and\ not keepValues: # delexicalisation v = '_' jsact['s2v'].append((s,v)) @@ -78,7 +81,7 @@ def format(self,dact): elif v=='?': # question case feature.append((s,v)) elif v=='_': # categories - if mem.has_key(s): # multiple feature values + if s in mem: # multiple feature values feature.append((s,v+str(mem[s]))) mem[s] += 1 else: # first occurance @@ -118,7 +121,7 @@ def format(self,dact): elif v=='?': # question case feature.append('SV-'+s+'=PENDING') elif v=='_': # categories - if mem.has_key(s): # multiple feature values + if s in mem: # multiple feature values feature.append('SV-'+s+'=VALUE'+str(mem[s])) mem[s] += 1 else: # first occurance @@ -136,13 +139,13 @@ def parse(self,dact,keepValues=False): #dadp = DialogActDelexicalizedParser() dadp = HardDActFormatter() - print dadp.format("inform(type='restaurant';count='182';area=dont_care)") - print dadp.format("reqmore()") - print dadp.format("request(area)") - print dadp.format("inform(name='fifth floor';address='hotel palomar 12 fourth street or rosie street')") - print dadp.format("inform(name='fifth floor';address='hotel palomar 12 fourth street and rosie street')") - print dadp.format("?select(food=dont_care;food='sea food')") - print dadp.format("?select(food='yes';food='no')") - print dadp.format("?select(battery rating=exceptional;battery rating=standard)") - print dadp.format("suggest(weight range=heavy;weight range=light weight;weightrange=dontcare)") - print dadp.format("?compare(name=satellite morpheus 36;warranty=1 year european;dimension=33.7 inch;name=tecra proteus 23;warranty=1 year international;dimension=27.4 inch)") + print(dadp.format("inform(type='restaurant';count='182';area=dont_care)")) + print(dadp.format("reqmore()")) + print(dadp.format("request(area)")) + print(dadp.format("inform(name='fifth floor';address='hotel palomar 12 fourth street or rosie street')")) + print(dadp.format("inform(name='fifth floor';address='hotel palomar 12 fourth street and rosie street')")) + print(dadp.format("?select(food=dont_care;food='sea food')")) + print(dadp.format("?select(food='yes';food='no')")) + print(dadp.format("?select(battery rating=exceptional;battery rating=standard)")) + print(dadp.format("suggest(weight range=heavy;weight range=light weight;weightrange=dontcare)")) + print(dadp.format("?compare(name=satellite morpheus 36;warranty=1 year european;dimension=33.7 inch;name=tecra proteus 23;warranty=1 year international;dimension=27.4 inch)")) diff --git a/loader/GentScorer.py b/loader/GentScorer.py index 395a31d..26f4d50 100644 --- a/loader/GentScorer.py +++ b/loader/GentScorer.py @@ -3,16 +3,14 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import os import json -import sys import math -import operator +from future.utils import iteritems from collections import Counter from nltk.util import ngrams -#from nltk.align.bleu import BLEU +file = open class ERRScorer(): @@ -48,7 +46,7 @@ def score(self,a,feat,gen): # exact match for categorical slots caty_slot_error = 0 # fo each slot - token pair in the detect pair dict - for s,tok in self.detectPairs['general'].iteritems(): + for s,tok in iteritems(self.detectPairs['general']): # token compare to comparetos = ['sv.'+s+'._1','sv.'+s+'._2','sv.'+s+'._3'] # count feature count in da feature @@ -68,7 +66,7 @@ def score(self,a,feat,gen): # key word match for binary slots, only an approximation bnay_slot_error = 0 # for each binary slot - for s,toks in self.detectPairs['binary'].iteritems(): + for s,toks in iteritems(self.detectPairs['binary']): # tokens compare to comparetos = ['sv.'+s+'.yes','sv.'+s+'.no', 'sv.'+s+'.dontcare','sv.'+s+'.none'] diff --git a/nn/NNGenerator.py b/nn/NNGenerator.py index cba4e5f..fb90bc7 100644 --- a/nn/NNGenerator.py +++ b/nn/NNGenerator.py @@ -9,10 +9,10 @@ import theano.gradient as G from collections import OrderedDict -from basic import * -from hlstm import * -from sclstm import * -from encdec import * +from .basic import * +from .hlstm import * +from .sclstm import * +from .encdec import * class NNGenerator(object): diff --git a/nn/encdec.py b/nn/encdec.py index 74db5ae..2ba220c 100644 --- a/nn/encdec.py +++ b/nn/encdec.py @@ -3,259 +3,266 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import operator +try: + + from Queue import PriorityQueue +except ImportError: + + from queue import PriorityQueue + import numpy as np +import operator import theano.tensor as T -from Queue import PriorityQueue +from .basic import * +from future.utils import iteritems from utils.mathUtil import softmax, sigmoid, tanh -from basic import * class encdec(BaseRLG): - def __init__(self, gentype, vocab, beamwidth, overgen, - vocab_size, hidden_size, batch_size, da_sizes): - + vocab_size, hidden_size, batch_size, da_sizes): + # calling superclass constructor BaseRLG.__init__(self, gentype, beamwidth, overgen, - vocab_size, hidden_size, batch_size, da_sizes) - self.da = self.dfs[1]-self.dfs[0] - self.ds = self.dfs[3]-self.dfs[2] - self.dv = self.dfs[4]-self.dfs[3] - self.vocab = vocab + vocab_size, hidden_size, batch_size, da_sizes) + self.da = self.dfs[1] - self.dfs[0] + self.ds = self.dfs[3] - self.dfs[2] + self.dv = self.dfs[4] - self.dfs[3] + self.vocab = vocab # init params self._init_params() def _init_params(self): - + # word embedding weight matrix - self.Wemb = theano.shared(0.3 * np.random.uniform(-1.0,1.0,\ - (self.di,self.dh)).astype(theano.config.floatX)) + self.Wemb = theano.shared(0.3 * np.random.uniform(-1.0, 1.0, \ + (self.di, self.dh)).astype(theano.config.floatX)) # DA embedding - npah = 0.3*np.random.uniform(-1.0,1.0,(self.da+1,self.dh)) - npsh = 0.3*np.random.uniform(-1.0,1.0,(self.ds+1,self.dh)) - npvh = 0.3*np.random.uniform(-1.0,1.0,(self.dv+1,self.dh)) - npah[self.da,:] = 0.0 - npsh[self.ds,:] = 0.0 - npvh[self.dv,:] = 0.0 - self.Wah = theano.shared(npah.astype(theano.config.floatX)) - self.Wsh = theano.shared(npsh.astype(theano.config.floatX)) - self.Wvh = theano.shared(npvh.astype(theano.config.floatX)) + npah = 0.3 * np.random.uniform(-1.0, 1.0, (self.da + 1, self.dh)) + npsh = 0.3 * np.random.uniform(-1.0, 1.0, (self.ds + 1, self.dh)) + npvh = 0.3 * np.random.uniform(-1.0, 1.0, (self.dv + 1, self.dh)) + npah[self.da, :] = 0.0 + npsh[self.ds, :] = 0.0 + npvh[self.dv, :] = 0.0 + self.Wah = theano.shared(npah.astype(theano.config.floatX)) + self.Wsh = theano.shared(npsh.astype(theano.config.floatX)) + self.Wvh = theano.shared(npvh.astype(theano.config.floatX)) # attention weights - self.Wha = theano.shared(0.3 * np.random.uniform(-1.0,1.0,\ - (self.dh*3,self.dh)).astype(theano.config.floatX)) - self.Vha = theano.shared(0.3 * np.random.uniform(-1.0,1.0,\ - (self.dh)).astype(theano.config.floatX)) + self.Wha = theano.shared(0.3 * np.random.uniform(-1.0, 1.0, \ + (self.dh * 3, self.dh)).astype(theano.config.floatX)) + self.Vha = theano.shared(0.3 * np.random.uniform(-1.0, 1.0, \ + (self.dh)).astype(theano.config.floatX)) # lstm gate weight matrix - self.Wgate = theano.shared(0.3 * np.random.uniform(-1.0,1.0,\ - (self.dh*3,self.dh*4)).astype(theano.config.floatX)) + self.Wgate = theano.shared(0.3 * np.random.uniform(-1.0, 1.0, \ + (self.dh * 3, self.dh * 4)).astype(theano.config.floatX)) # hidden to output weight matrix - self.Who= theano.shared(0.3 * np.random.uniform(-1.0,1.0,\ - (self.dh,self.di)).astype(theano.config.floatX)) + self.Who = theano.shared(0.3 * np.random.uniform(-1.0, 1.0, \ + (self.dh, self.di)).astype(theano.config.floatX)) # initial memory cell and hidden layer - self.h0 = theano.shared(np.zeros((self.db,self.dh), - dtype=theano.config.floatX)) - self.c0 = theano.shared(np.zeros((self.db,self.dh), - dtype=theano.config.floatX)) + self.h0 = theano.shared(np.zeros((self.db, self.dh), + dtype=theano.config.floatX)) + self.c0 = theano.shared(np.zeros((self.db, self.dh), + dtype=theano.config.floatX)) # all parameters - self.params = [ - self.Wemb, - self.Wah, self.Wsh, self.Wvh, - self.Wha, self.Vha, - self.Wgate, - self.Who ] - - def setWordVec(self,word2vec): + self.params = [ + self.Wemb, + self.Wah, self.Wsh, self.Wvh, + self.Wha, self.Vha, + self.Wgate, + self.Who] + + def setWordVec(self, word2vec): self.Wemb_np = self.Wemb.get_value() - for w,v in word2vec.iteritems(): - self.Wemb_np[w,:] = v + for w, v in iteritems(word2vec): + self.Wemb_np[w, :] = v self.Wemb.set_value(self.Wemb_np) - def _emb(self, a, s, v): - a_emb = T.sum(self.Wah[a,:],axis=0) - s_emb = self.Wsh[s,:] - v_emb = self.Wvh[v,:] - sv_emb= s_emb + v_emb + a_emb = T.sum(self.Wah[a, :], axis=0) + s_emb = self.Wsh[s, :] + v_emb = self.Wvh[v, :] + sv_emb = s_emb + v_emb return a_emb, sv_emb - def unroll(self,a,s,v,words,cutoff_f,cutoff_b): - + def unroll(self, a, s, v, words, cutoff_f, cutoff_b): + # embed DA - [a_emb,sv_emb],_= theano.scan(fn=self._emb, - sequences=[a,s,v]) - sv_emb = sv_emb.dimshuffle(1,0,2) + [a_emb, sv_emb], _ = theano.scan(fn=self._emb, + sequences=[a, s, v]) + sv_emb = sv_emb.dimshuffle(1, 0, 2) # recurrence - [h,c,p],_ = theano.scan(fn=self._recur, - sequences=[words[:-1,:],words[1:,:]], - outputs_info=[self.h0,self.c0,None], - non_sequences=[a_emb,sv_emb]) + [h, c, p], _ = theano.scan(fn=self._recur, + sequences=[words[:-1, :], words[1:, :]], + outputs_info=[self.h0, self.c0, None], + non_sequences=[a_emb, sv_emb]) # compute desired sent_logp by slicing - cutoff_logp = collectSentLogp(p,cutoff_f[4],cutoff_b) - cost = -T.sum(cutoff_logp) + cutoff_logp = collectSentLogp(p, cutoff_f[4], cutoff_b) + cost = -T.sum(cutoff_logp) return cost, cutoff_logp def _attend(self, sv_emb_x, wv_t, h_tm1): - state_x = T.concatenate([wv_t,h_tm1,sv_emb_x],axis=1) - score_x = T.dot(T.tanh(T.dot(state_x,self.Wha)),self.Vha) + state_x = T.concatenate([wv_t, h_tm1, sv_emb_x], axis=1) + score_x = T.dot(T.tanh(T.dot(state_x, self.Wha)), self.Vha) return score_x def _recur(self, w_t, y_t, h_tm1, c_tm1, a_emb, sv_emb): - + # input word embedding - wv_t = T.nnet.sigmoid(self.Wemb[w_t,:]) + wv_t = T.nnet.sigmoid(self.Wemb[w_t, :]) # attention - b_t,_= theano.scan(fn=self._attend, - sequences=[sv_emb], - non_sequences=[wv_t,h_tm1]) - b_t = T.nnet.softmax(b_t.dimshuffle(1,0)) - sv_emb_t = T.tensordot(b_t,sv_emb,[[1,0],[0,1]]) - da_emb_t = T.tanh( a_emb+sv_emb_t ) + b_t, _ = theano.scan(fn=self._attend, + sequences=[sv_emb], + non_sequences=[wv_t, h_tm1]) + b_t = T.nnet.softmax(b_t.dimshuffle(1, 0)) + sv_emb_t = T.tensordot(b_t, sv_emb, [[1, 0], [0, 1]]) + da_emb_t = T.tanh(a_emb + sv_emb_t) # compute ig, fg, og together and slice it - gates_t = T.dot( T.concatenate([wv_t,h_tm1,da_emb_t],axis=1),self.Wgate) - ig = T.nnet.sigmoid(gates_t[:,:self.dh]) - fg = T.nnet.sigmoid(gates_t[:,self.dh:self.dh*2]) - og = T.nnet.sigmoid(gates_t[:,self.dh*2:self.dh*3]) - cx_t= T.tanh( gates_t[:,self.dh*3:] ) + gates_t = T.dot(T.concatenate([wv_t, h_tm1, da_emb_t], axis=1), self.Wgate) + ig = T.nnet.sigmoid(gates_t[:, :self.dh]) + fg = T.nnet.sigmoid(gates_t[:, self.dh:self.dh * 2]) + og = T.nnet.sigmoid(gates_t[:, self.dh * 2:self.dh * 3]) + cx_t = T.tanh(gates_t[:, self.dh * 3:]) # update lstm internal state - c_t = ig*cx_t + fg*c_tm1 + c_t = ig * cx_t + fg * c_tm1 # obtain new hiddne layer - h_t = og*T.tanh(c_t) + h_t = og * T.tanh(c_t) # compute output distribution target word prob - o_t = T.nnet.softmax( T.dot(h_t,self.Who) ) - p_t = o_t[T.arange(self.db),y_t] + o_t = T.nnet.softmax(T.dot(h_t, self.Who)) + p_t = o_t[T.arange(self.db), y_t] return h_t, c_t, p_t - def _npemb(self,a,s,v): - return np.sum(self.Wah_np[a,:],axis=0),\ - self.Wsh_np[s,:]+self.Wvh_np[v,:] + def _npemb(self, a, s, v): + return np.sum(self.Wah_np[a, :], axis=0), \ + self.Wsh_np[s, :] + self.Wvh_np[v, :] - def beamSearch(self,a,s,v): + def beamSearch(self, a, s, v): # embed DA - a_emb, sv_emb = self._npemb(a,s,v) + a_emb, sv_emb = self._npemb(a, s, v) # end nodes endnodes = [] # initial layers - h0,c0 = np.zeros(self.dh),np.zeros(self.dh) + h0, c0 = np.zeros(self.dh), np.zeros(self.dh) # starting node - node = BeamSearchNode(h0,c0,None,1,0,1) + node = BeamSearchNode(h0, c0, None, 1, 0, 1) node.sv = sv_emb - node.a = a_emb + node.a = a_emb # queue for beam search - nodes= PriorityQueue() - nodes.put((-node.eval(),node)) + nodes = PriorityQueue() + nodes.put((-node.eval(), node)) qsize = 1 # start beam search while True: # give up when decoding takes too long - if qsize>10000: break + if qsize > 10000: break # fetch the best node score, n = nodes.get() # if end of sentence token - if n.wordid==1 and n.prevNode!=None: - endnodes.append((score,n)) + if n.wordid == 1 and n.prevNode != None: + endnodes.append((score, n)) # if reach maximum # of sentences required - if len(endnodes)>=self.overgen: break - else: continue + if len(endnodes) >= self.overgen: + break + else: + continue # decode for one step using decoder words, probs, c, h = self._gen(n) # put them into a queue for i in range(len(words)): - node = BeamSearchNode(h,c,n,words[i], - n.logp+np.log10(probs[i]),n.leng+1) + node = BeamSearchNode(h, c, n, words[i], + n.logp + np.log10(probs[i]), n.leng + 1) node.sv = sv_emb - node.a = a_emb - nodes.put( (-node.eval(),node) ) + node.a = a_emb + nodes.put((-node.eval(), node)) # increase qsize - qsize += len(words)-1 + qsize += len(words) - 1 # if no finished nodes, choose the top scored paths - if len(endnodes)==0: + if len(endnodes) == 0: endnodes = [nodes.get() for n in range(self.overgen)] # choose nbest paths, back trace them utts = [] - for score,n in sorted(endnodes,key=operator.itemgetter(0)): + for score, n in sorted(endnodes, key=operator.itemgetter(0)): utt = [n.wordid] - while n.prevNode!=None: + while n.prevNode != None: # back trace n = n.prevNode utt.append(n.wordid) utt = utt[::-1] - utts.append((score,utt)) + utts.append((score, utt)) return utts - def sample(self,a,s,v): + def sample(self, a, s, v): # embed DA - a_emb, sv_emb = self._npemb(a,s,v) + a_emb, sv_emb = self._npemb(a, s, v) # initial state - h0,c0 = np.zeros(self.dh),np.zeros(self.dh) + h0, c0 = np.zeros(self.dh), np.zeros(self.dh) # container gens = [] # to obtain topk generations for i in range(self.overgen): # starting node - node = BeamSearchNode(h0,c0,None,1,0,1) + node = BeamSearchNode(h0, c0, None, 1, 0, 1) node.sv = sv_emb - node.a = a_emb + node.a = a_emb # put in queue nodes = [node] # start sampling while True: # check stopping criteria last_node = nodes[-1] - if last_node.wordid==1 and len(nodes)>1: + if last_node.wordid == 1 and len(nodes) > 1: break - if len(nodes)>40: # undesirable long utt + if len(nodes) > 40: # undesirable long utt break # expand for one time step words, probs, c, h = self._gen(last_node) # sampling according to probability - o_sample = np.argmax(np.random.multinomial(1,probs,1)) + o_sample = np.argmax(np.random.multinomial(1, probs, 1)) # put new node into the queue - node = BeamSearchNode(h,c,last_node,words[o_sample], - last_node.logp+np.log10(probs[o_sample]),last_node.leng+1) + node = BeamSearchNode(h, c, last_node, words[o_sample], + last_node.logp + np.log10(probs[o_sample]), last_node.leng + 1) node.sv = sv_emb - node.a = a_emb - nodes.append( node ) + node.a = a_emb + nodes.append(node) # obtain sentence gen = [n.wordid for n in nodes] # score the sentences score = -nodes[-1].eval() # make sure the generated sentence doesn't repeat - if (score,gen) not in gens: - gens.append((score,gen)) + if (score, gen) not in gens: + gens.append((score, gen)) # ranking generation according to score - overgen = self.overgen if len(gens)>self.overgen else len(gens) - gens = sorted(gens,key=operator.itemgetter(0))[:overgen] - return gens + overgen = self.overgen if len(gens) > self.overgen else len(gens) + gens = sorted(gens, key=operator.itemgetter(0))[:overgen] + return gens + + def _gen(self, node): - def _gen(self,node): - # input word embedding - wv_t = sigmoid(self.Wemb_np[node.wordid,:]) + wv_t = sigmoid(self.Wemb_np[node.wordid, :]) # attention b_t = np.zeros((node.sv.shape[0])) for j in range(node.sv.shape[0]): b_t[j] = np.dot(tanh(np.dot( - np.concatenate([wv_t,node.h,node.sv[j]],axis=0), - self.Wha_np)),self.Vha_np) + np.concatenate([wv_t, node.h, node.sv[j]], axis=0), + self.Wha_np)), self.Vha_np) b_t = softmax(b_t) - sv_emb_t = np.dot(b_t,node.sv) - da_emb_t = tanh( node.a+sv_emb_t ) + sv_emb_t = np.dot(b_t, node.sv) + da_emb_t = tanh(node.a + sv_emb_t) # compute ig, fg, og together and slice it - gates_t = np.dot( np.concatenate([wv_t,node.h,da_emb_t],axis=0), - self.Wgate_np) - ig = sigmoid(gates_t[:self.dh]) - fg = sigmoid(gates_t[self.dh:self.dh*2]) - og = sigmoid(gates_t[self.dh*2:self.dh*3]) - cx_t= tanh( gates_t[self.dh*3:] ) + gates_t = np.dot(np.concatenate([wv_t, node.h, da_emb_t], axis=0), + self.Wgate_np) + ig = sigmoid(gates_t[:self.dh]) + fg = sigmoid(gates_t[self.dh:self.dh * 2]) + og = sigmoid(gates_t[self.dh * 2:self.dh * 3]) + cx_t = tanh(gates_t[self.dh * 3:]) # update lstm internal state - c_t = np.multiply(ig,cx_t) + np.multiply(fg,node.c) + c_t = np.multiply(ig, cx_t) + np.multiply(fg, node.c) # obtain new hiddne layer - h_t = np.multiply(og,tanh(c_t)) + h_t = np.multiply(og, tanh(c_t)) # compute output distribution target word prob - o_t = softmax( np.dot(h_t,self.Who_np) ) + o_t = softmax(np.dot(h_t, self.Who_np)) # make sure we won't sample unknown word o_t[0] = 0.0 selected_words = np.argsort(o_t)[::-1][:self.beamwidth].tolist() @@ -263,13 +270,11 @@ def _gen(self,node): return selected_words, o_t[selected_words], c_t, h_t def loadConverseParams(self): - self.Wemb_np = self.params[0].get_value() - self.Wah_np = self.params[1].get_value() - self.Wsh_np = self.params[2].get_value() - self.Wvh_np = self.params[3].get_value() - self.Wha_np = self.params[4].get_value() - self.Vha_np = self.params[5].get_value() - self.Wgate_np = self.params[6].get_value() - self.Who_np = self.params[7].get_value() - - + self.Wemb_np = self.params[0].get_value() + self.Wah_np = self.params[1].get_value() + self.Wsh_np = self.params[2].get_value() + self.Wvh_np = self.params[3].get_value() + self.Wha_np = self.params[4].get_value() + self.Vha_np = self.params[5].get_value() + self.Wgate_np = self.params[6].get_value() + self.Who_np = self.params[7].get_value() diff --git a/nn/hlstm.py b/nn/hlstm.py index 5fbbae0..a6adef3 100644 --- a/nn/hlstm.py +++ b/nn/hlstm.py @@ -6,12 +6,17 @@ import operator import numpy as np import theano.tensor as T -from Queue import PriorityQueue +from future.utils import iteritems +try: + + from Queue import PriorityQueue +except ImportError: + + from queue import PriorityQueue from utils.mathUtil import softmax, sigmoid, tanh -from theano.ifelse import ifelse from copy import deepcopy -from basic import * +from .basic import * class hlstm(BaseRLG): @@ -59,7 +64,7 @@ def _init_params(self): def setWordVec(self,word2vec): self.Wemb_np = self.Wemb.get_value() - for w,v in word2vec.iteritems(): + for w, v in iteritems(word2vec): self.Wemb_np[w,:] = v self.Wemb.set_value(self.Wemb_np) diff --git a/nn/ngmodel.py b/nn/ngmodel.py index 686b053..b7d13f3 100644 --- a/nn/ngmodel.py +++ b/nn/ngmodel.py @@ -5,10 +5,14 @@ ###################################################################### import operator import numpy as np -from Queue import PriorityQueue -from utils.mathUtil import softmax, sigmoid, tanh -from copy import deepcopy -from basic import * +from future.utils import iteritems +try: + + from Queue import PriorityQueue +except ImportError: + + from queue import PriorityQueue +from .basic import * class NGModel(object): @@ -58,14 +62,14 @@ def train(self,sents): # normalise everything into log prob sum1 = sum([cnt for cnt in self.model[1].values()]) - for w,cnt in self.model[1].iteritems(): + for w, cnt in iteritems(self.model[1]): self.model[1][w] = cnt/sum1 self.model[1] = sorted(self.model[1].items(), key=operator.itemgetter(1),reverse=True) for n in range(2,self.n+1): - for context,wdct in self.model[n].iteritems(): + for context, wdct in iteritems(self.model[n]): sumc = sum([cnt for cnt in wdct.values()]) - for w,cnt in wdct.iteritems(): + for w, cnt in iteritems(wdct): self.model[n][context][w] = cnt/sumc self.model[n][context] = sorted( self.model[n][context].items(), diff --git a/nn/sclstm.py b/nn/sclstm.py index d6d58b4..0634397 100644 --- a/nn/sclstm.py +++ b/nn/sclstm.py @@ -6,10 +6,16 @@ import operator import numpy as np import theano.tensor as T -from Queue import PriorityQueue +from future.utils import iteritems +try: + + from Queue import PriorityQueue +except ImportError: + + from queue import PriorityQueue from utils.mathUtil import softmax, sigmoid, tanh -from basic import * +from .basic import * class sclstm(BaseRLG): @@ -60,7 +66,7 @@ def _init_params(self): def setWordVec(self,word2vec): self.Wemb_np = self.Wemb.get_value() - for w,v in word2vec.iteritems(): + for w, v in iteritems(word2vec): self.Wemb_np[w,:] = v self.Wemb.set_value(self.Wemb_np) diff --git a/utils/nlp.py b/utils/nlp.py index 4634699..dc7aaeb 100644 --- a/utils/nlp.py +++ b/utils/nlp.py @@ -3,96 +3,94 @@ # Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 # ###################################################################### ###################################################################### -import os -import json -import sys -import operator import re +file = open + fin = file('utils/nlp/mapping.pair') replacements = [] for line in fin.readlines(): - tok_from, tok_to = line.replace('\n','').split('\t') - replacements.append((' '+tok_from+' ',' '+tok_to+' ')) + tok_from, tok_to = line.replace('\n', '').split('\t') + replacements.append((' ' + tok_from + ' ', ' ' + tok_to + ' ')) + -def insertSpace(token,text): +def insertSpace(token, text): sidx = 0 while True: - sidx = text.find(token,sidx) - if sidx==-1: + sidx = text.find(token, sidx) + if sidx == -1: break - if sidx+1@]','',text) - #text = re.sub('[\":\<>@\(\)]','',text) - text = text.replace(' - ','') + text = re.sub('[\":\<>@]', '', text) + # text = re.sub('[\":\<>@\(\)]','',text) + text = text.replace(' - ', '') # insert white space before and after tokens: - for token in ['?','.',',','!']: - text = insertSpace(token,text) - + for token in ['?', '.', ',', '!']: + text = insertSpace(token, text) + # replace it's, does't, you'd ... etc - text = re.sub('^\'','',text) - text = re.sub('\'$','',text) - text = re.sub('\'\s',' ',text) - text = re.sub('\s\'',' ',text) + text = re.sub('^\'', '', text) + text = re.sub('\'$', '', text) + text = re.sub('\'\s', ' ', text) + text = re.sub('\s\'', ' ', text) for fromx, tox in replacements: - text = ' '+text+' ' - text = text.replace(fromx,tox)[1:-1] + text = ' ' + text + ' ' + text = text.replace(fromx, tox)[1:-1] # insert white space for 's - text = insertSpace('\'s',text) + text = insertSpace('\'s', text) # remove multiple spaces - text = re.sub(' +',' ',text) - + text = re.sub(' +', ' ', text) + # concatenate numbers tmp = text tokens = text.split() i = 1 - while i