最近在做QA系统，用tensorflow做了些实验，下面的的是一个cnn的评分网络。主要参考了《APPLYING DEEP LEARNING TO ANSWER SELECTION: A STUDY AND AN OPEN TASK》这篇论文与wildml博客中的一篇文章。
import tensorflow as tf
import numpy as np
class QaCNN():
def __init__(self , batchsize , sequencesize , vecsize , outsize , filtersizes , num_filters):
self.vecsize = vecsize
self.outsize = outsize
self.batchsize = batchsize
self.sequencesize = sequencesize
self.question = tf.placeholder(tf.float32 , [None , vecsize * sequencesize],name='question')
self.answer_rigth = tf.placeholder(tf.float32 , [None , vecsize * sequencesize],name='answer_right')
self.answer_wrong = tf.placeholder(tf.float32 , [None , vecsize * sequencesize],name='answer_wrong')
tenQ = tf.reshape(self.question , [-1 , self.sequencesize , self.vecsize , 1])
tenR = tf.reshape(self.answer_rigth , [-1 , self.sequencesize , self.vecsize , 1])
tenW = tf.reshape(self.answer_wrong , [-1 , self.sequencesize , self.vecsize , 1])
tensorResultQ = []
tensorResultR = []
tensorResultW = []
for i , filtersize in enumerate(filtersizes):
with tf.name_scope("conv-maxpool-%s" % filtersize):
filter_shape = [filtersize, self.vecsize, 1, num_filters]
#W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1),name='W')
W = tf.get_variable(initializer=tf.truncated_normal(filter_shape, stddev=0.1),
name="W-%s" % str(filtersize))
#b = tf.Variable(tf.constant(0.1, shape=[num_filters]),name='b')
b = tf.get_variable(initializer=tf.constant(0.1, shape=[num_filters]),
name="b-%s" % str(filtersize))
pooledQ = self.conv2dPool(tenQ , W , b , filtersize)
pooledR = self.conv2dPool(tenR, W , b , filtersize)
pooledW = self.conv2dPool(tenW , W , b , filtersize)
tensorResultQ.append(pooledQ)
tensorResultR.append(pooledR)
tensorResultW.append(pooledW)
flat_length = len(filtersizes) * num_filters
tenQ_flat = tf.reshape(tf.concat(tensorResultQ,3),[-1,flat_length])
tenR_flat = tf.reshape(tf.concat(tensorResultR,3),[-1,flat_length])
tenW_flat = tf.reshape(tf.concat(tensorResultW,3),[-1,flat_length])
exy = tf.reduce_sum(tf.multiply(tenQ_flat , tenR_flat) , 1)
x = tf.sqrt(tf.reduce_sum(tf.multiply(tenQ_flat , tenQ_flat) , 1))
y = tf.sqrt(tf.reduce_sum(tf.multiply(tenR_flat , tenR_flat) , 1))
cosineQR = tf.div(exy , tf.multiply(x , y),name = 'cosineQR')
exy = tf.reduce_sum(tf.multiply(tenQ_flat , tenW_flat) , 1)
x = tf.sqrt(tf.reduce_sum(tf.multiply(tenQ_flat , tenQ_flat) , 1))
y = tf.sqrt(tf.reduce_sum(tf.multiply(tenW_flat , tenW_flat) , 1))
cosineQW = tf.div(exy , tf.multiply(x , y),name = 'cosineQW')
with tf.name_scope('losses'):
zero = tf.constant(0, shape=[self.batchsize], dtype=tf.float32)
margin = tf.constant(0.05, shape=[self.batchsize], dtype=tf.float32)
self.losses = tf.maximum(zero, tf.subtract(margin, tf.subtract(cosineQR, cosineQW)),name = 'loss_tensor')
self.loss = tf.reduce_sum(self.losses,name='loss')
with tf.name_scope('acc'):
self.correct = tf.equal(zero,self.losses)
self.accuracy = tf.reduce_mean(tf.cast(self.correct , 'float'),name='accuracy')
tf.summary.scalar('loss',self.loss)
self.variable_summaries(self.accuracy)
self.merged = tf.summary.merge_all()
def variable_summaries(self , var):
'''Attach a lot of summaries to a Tensor (for TensorBoard visualization).'''
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean',mean)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev',stddev)
tf.summary.scalar('max',tf.reduce_max(var))
tf.summary.scalar('min',tf.reduce_min(var))
tf.summary.histogram('histogram',var)
def conv2dPool(self ,x,W,b,filtersize):
conv = tf.nn.conv2d(x , W , strides =[1,1,1,1],padding='VALID')
h = tf.nn.relu(tf.nn.bias_add(conv ,b))
pooled = tf.nn.max_pool(h,ksize=[1,self.sequencesize - filtersize + 1 , 1, 1],strides=[1,1,1,1],padding='VALID')
return pooled
import numpy as np
import time
import os
import tensorflow as tf
from qacnn_g import *
from process import *
batchsize = 100
sequencesize = 10
vecsize = 200
outsize = 10
root = './lib/'
filtersize = [1,2,3,5]
num_filter = 500
if os.path.exists('./lib/corpus.seg.length.out'):
os.remove(root + 'corpus.seg.length.out')
logfolder = time.strftime("%Y%m%d_%H%M%S", time.localtime())
cnn = QaCNN(batchsize , sequencesize , vecsize , outsize , filtersize , num_filter)
train_step = tf.train.AdamOptimizer(1e-4).minimize(cnn.loss)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter('./LOGS/'+logfolder,sess.graph)
saver = tf.train.Saver(tf.all_variables())
print ('init...')
dataprocess = DataProcess(root + 'word2vec.bin')
dataprocess.normalize(root + 'corpus.seg.out', root + 'corpus.seg.length.out')
dataprocess.initdata(root + 'corpus.seg.length.out')
start = time.time()
for i in range(120000):
batch =dataprocess.nextbatch(batchsize)#ake_data(batchsize) #fake_data(20, True)
sess.run(train_step ,feed_dict={cnn.question: batch[0],cnn.answer_rigth:batch[1] , cnn.answer_wrong:batch[2]})
if i % 10 == 0:
summary,loss, accuracy,_ = sess.run([cnn.merged , cnn.loss , cnn.accuracy , train_step ] , {cnn.question: batch[0],cnn.answer_rigth:batch[1] , cnn.answer_wrong:batch[2]})
train_writer.add_summary(summary , i)
end = time.time()
elapse = (end - start)
print ('iterator %d.\tloss=%f\taccuracy=%f\telapse=%f'%(i,loss,accuracy,elapse))
start = time.time()
else:
sess.run(train_step ,feed_dict={cnn.question: batch[0],cnn.answer_rigth:batch[1] , cnn.answer_wrong:batch[2]})
train_writer.close()
saver.save(sess , './model/qa.cnn')
sess.close()
print ('end...')
使用了1200W数据来训练，下面是loss的拆线图。
accuracy
使用模型进行打分，这里取了cosineR这个正向答案与问题的cosine值进行度量。
# -*- coding: utf-8 -*-
import tensorflow as tf
import numpy as np
import os
import time
from process import *
tf.flags.DEFINE_string('./lib/corpus.out','','Data to predict')
tf.flags.DEFINE_string('checkpoint_dir','./model/','checkpoint directory from training run')
tf.flags.DEFINE_integer('batch_size',1000,'batch size')
tf.flags.DEFINE_string('root','./lib','root dir')
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print ('\nParameters:')
for attr , value in sorted(FLAGS.__flags.items()):
print ('{}={}'.format(attr.upper() , value))
print('')
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
dataprocess = DataProcess('./lib/word2vec.bin')
dataprocess.initdata('./lib/corpus.seg.length.out')
graph = tf.Graph()
with graph.as_default():
sess = tf.Session()
with sess.as_default():
saver = tf.train.import_meta_graph('{}.meta'.format(checkpoint_file))
saver.restore(sess , checkpoint_file)
start = time.time()
bestAnswer = None
maxScore = -1.0
for i in range(10000):
question = graph.get_operation_by_name('question').outputs[0]
answer_right = graph.get_operation_by_name('answer_right').outputs[0]
answer_wrong = graph.get_operation_by_name('answer_wrong').outputs[0]
loss = graph.get_operation_by_name('losses/loss').outputs[0]
cosineQR = graph.get_operation_by_name('cosineQR').outputs[0]
questionbatch = dataprocess.getSentenceVec('你叫什么名字',10,FLAGS.batch_size)
batchs = dataprocess.nextbatch(FLAGS.batch_size)
cosineQR = sess.run(cosineQR, {question:questionbatch, answer_right:batchs[1],answer_wrong:batchs[2]})
ndx = np.argmax(cosineQR)
score = cosineQR[ndx]
if maxScore < score:
maxScore = score
bestAnswer = batchs[3][ndx]
print ('iterate : %d\tscore:%f\tmaxscore:%f\tanswer:%s'%(i,score,maxScore,(batchs[3][ndx]).strip('\n')))
end = time.time()
print('time used:%f'%(end - start))
print('maxScore:%f'%maxScore)
print('best answer :%s'%bestAnswer)
def find(cosineTensor):
return np.argmax(cosineTensor)