from collections import OrderedDict

import copy

import cPickle

import gzip

import os

import urllib

import random

import stat

import subprocess

import sys

import time

import numpy

import theano

from theano import tensor as T

PREFIX = os.getenv('ATISDATA', 'data')

def shuffle(lol, seed):

'''

for l in lol:

random.seed(seed)

random.shuffle(l)

def contextwin(l, win):

'''

assert (win % 2) == 1

assert win >= 1

l = list(l)

lpadded = win//2 * [-1] + l + win//2 * [-1]

out = [lpadded[i:i+win] for i in range(len(l))]

assert len(out) == len(l)

return out

def atisfold(fold):

assert fold in range(5)

filename = os.path.join(PREFIX, 'atis.fold'+str(fold)+'.pkl.gz')

f = gzip.open(filename, 'rb')

train_set, valid_set, test_set, dicts = cPickle.load(f)

return train_set, valid_set, test_set, dicts

def conlleval(p, g, w, filename):

'''

out = ''

for sl, sp, sw in zip(g, p, w):

out += 'BOS O O\n'

for wl, wp, w in zip(sl, sp, sw):

out += w + ' ' + wl + ' ' + wp + '\n'

out += 'EOS O O\n\n'

f = open(filename, 'w')

f.writelines(out)

f.close()

return get_perf(filename)

def download(origin):

'''

print 'Downloading data from %s' % origin

name = origin.split('/')[-1]

urllib.urlretrieve(origin, name)

def get_perf(filename):

''' run conlleval.pl perl script to obtain

_conlleval = 'conlleval.pl'

if not os.path.isfile(_conlleval):

url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl'

download(url)

os.chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions

proc = subprocess.Popen(["perl",

_conlleval],

stdin=subprocess.PIPE,

stdout=subprocess.PIPE)

stdout, _ = proc.communicate(''.join(open(filename).readlines()))

for line in stdout.split('\n'):

if 'accuracy' in line:

out = line.split()

break

precision = float(out[6][:-2])

recall = float(out[8][:-2])

f1score = float(out[10])

return {'p': precision, 'r': recall, 'f1': f1score}

class RNNSLU(object):

''' elman neural net model '''

def __init__(self, nh, nc, ne, de, cs):

'''

# parameters of the model

self.emb = theano.shared(name='embeddings',

value=0.2 * numpy.random.uniform(-1.0, 1.0,

(ne+1, de))

# add one for padding at the end

.astype(theano.config.floatX))

self.wx = theano.shared(name='wx',

value=0.2 * numpy.random.uniform(-1.0, 1.0,

(de * cs, nh))

.astype(theano.config.floatX))

self.wh = theano.shared(name='wh',

value=0.2 * numpy.random.uniform(-1.0, 1.0,

(nh, nh))

.astype(theano.config.floatX))

self.w = theano.shared(name='w',

value=0.2 * numpy.random.uniform(-1.0, 1.0,

(nh, nc))

.astype(theano.config.floatX))

self.bh = theano.shared(name='bh',

value=numpy.zeros(nh,

dtype=theano.config.floatX))

self.b = theano.shared(name='b',

value=numpy.zeros(nc,

dtype=theano.config.floatX))

self.h0 = theano.shared(name='h0',

value=numpy.zeros(nh,

dtype=theano.config.floatX))

# bundle

self.params = [self.emb, self.wx, self.wh, self.w,

self.bh, self.b, self.h0]

# end-snippet-2

# as many columns as context window size

# as many lines as words in the sentence

# start-snippet-3

idxs = T.imatrix()

x = self.emb[idxs].reshape((idxs.shape[0], de*cs))

y_sentence = T.ivector('y_sentence') # labels

# end-snippet-3 start-snippet-4

def recurrence(x_t, h_tm1):

h_t = T.nnet.sigmoid(T.dot(x_t, self.wx)

+ T.dot(h_tm1, self.wh) + self.bh)

s_t = T.nnet.softmax(T.dot(h_t, self.w) + self.b)

return [h_t, s_t]

[h, s], _ = theano.scan(fn=recurrence,

sequences=x,

outputs_info=[self.h0, None],

n_steps=x.shape[0])

p_y_given_x_sentence = s[:, 0, :]

y_pred = T.argmax(p_y_given_x_sentence, axis=1)

# end-snippet-4

# cost and gradients and learning rate

# start-snippet-5

lr = T.scalar('lr')

sentence_nll = -T.mean(T.log(p_y_given_x_sentence)

[T.arange(x.shape[0]), y_sentence])

sentence_gradients = T.grad(sentence_nll, self.params)

sentence_updates = OrderedDict((p, p - lr*g)

# end-snippet-5

for p, g in

zip(self.params, sentence_gradients))

# theano functions to compile

# start-snippet-6

self.classify = theano.function(inputs=[idxs], outputs=y_pred)

self.sentence_train = theano.function(inputs=[idxs, y_sentence, lr],

outputs=sentence_nll,

updates=sentence_updates)

# end-snippet-6 start-snippet-7

self.normalize = theano.function(inputs=[],

updates={self.emb:

self.emb /

T.sqrt((self.emb**2)

.sum(axis=1))

.dimshuffle(0, 'x')})

# end-snippet-7

def train(self, x, y, window_size, learning_rate):

cwords = contextwin(x, window_size)

words = map(lambda x: numpy.asarray(x).astype('int32'), cwords)

labels = y

self.sentence_train(words, labels, learning_rate)

self.normalize()

def save(self, folder):

for param in self.params:

numpy.save(os.path.join(folder,

param.name + '.npy'), param.get_value())

def load(self, folder):

for param in self.params:

param.set_value(numpy.load(os.path.join(folder,

param.name + '.npy')))

def main(param=None):

if not param:

param = {'fold': 3,

# 5 folds 0,1,2,3,4

'data': 'atis',

'lr': 0.0970806646812754,

'verbose': 1,

'decay': True,

# decay on the learning rate if improvement stops

'win': 7,

# number of words in the context window

'nhidden': 200,

# number of hidden units

'seed': 345,

'emb_dimension': 50,

# dimension of word embedding

'nepochs': 60,

# 60 is recommended

'savemodel': False}

print param

folder = os.path.basename(__file__).split('.')[0]

if not os.path.exists(folder):

os.mkdir(folder)

# load the dataset

train_set, valid_set, test_set, dic = atisfold(param['fold'])

idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())

idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())

train_lex, train_ne, train_y = train_set

valid_lex, valid_ne, valid_y = valid_set

test_lex, test_ne, test_y = test_set

vocsize = len(set(reduce(lambda x, y: list(x) + list(y),

train_lex + valid_lex + test_lex)))

nclasses = len(set(reduce(lambda x, y: list(x)+list(y),

train_y + test_y + valid_y)))

nsentences = len(train_lex)

groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]

words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]

groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]

words_test = [map(lambda x: idx2word[x], w) for w in test_lex]

# instanciate the model

numpy.random.seed(param['seed'])

random.seed(param['seed'])

rnn = RNNSLU(nh=param['nhidden'],

nc=nclasses,

ne=vocsize,

de=param['emb_dimension'],

cs=param['win'])

# train with early stopping on validation set

best_f1 = -numpy.inf

param['clr'] = param['lr']

for e in xrange(param['nepochs']):

# shuffle

shuffle([train_lex, train_ne, train_y], param['seed'])

param['ce'] = e

tic = time.time()

for i, (x, y) in enumerate(zip(train_lex, train_y)):

rnn.train(x, y, param['win'], param['clr'])

print '[learning] epoch %i >> %2.2f%%'%(e,(i+1)*100./nsentences),'completed in %.2f (sec) <<\r'%(time.time()-tic),

sys.stdout.flush()

# evaluation // back into the real world : idx -> words

predictions_test = [map(lambda x: idx2label[x],

rnn.classify(numpy.asarray(

contextwin(x, param['win'])).astype('int32')))

for x in test_lex]

predictions_valid = [map(lambda x: idx2label[x],

rnn.classify(numpy.asarray(

contextwin(x, param['win'])).astype('int32')))

for x in valid_lex]

# evaluation // compute the accuracy using conlleval.pl

res_test = conlleval(predictions_test,

groundtruth_test,

words_test,

folder + '/current.test.txt')

res_valid = conlleval(predictions_valid,

groundtruth_valid,

words_valid,

folder + '/current.valid.txt')

if res_valid['f1'] > best_f1:

if param['savemodel']:

rnn.save(folder)

best_rnn = copy.deepcopy(rnn)

best_f1 = res_valid['f1']

if param['verbose']:

print('NEW BEST: epoch', e,

'valid F1', res_valid['f1'],

'best test F1', res_test['f1'])

param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1']

param['vp'], param['tp'] = res_valid['p'], res_test['p']

param['vr'], param['tr'] = res_valid['r'], res_test['r']

param['be'] = e

subprocess.call(['mv', folder + '/current.test.txt',

folder + '/best.test.txt'])

subprocess.call(['mv', folder + '/current.valid.txt',

folder + '/best.valid.txt'])

else:

if param['verbose']:

print ''

# learning rate decay if no improvement in 10 epochs

if param['decay'] and abs(param['be']-param['ce']) >= 10:

param['clr'] *= 0.5

rnn = best_rnn

if param['clr'] < 1e-5:

break

print('BEST RESULT: epoch', param['be'],

'valid F1', param['vf1'],

'best test F1', param['tf1'],

'with the model', folder)

if __name__ == '__main__':

main()