很遗憾没有在规定的时间点（2018-9-25 12:00:00）完成所有的功能并上传数据，只做到写了模型代码并只跑了一轮迭代，现将代码部分贴出。
import keras
from keras.layers import Conv2D, MaxPooling2D, Flatten, Conv3D, MaxPooling3D
from keras.layers import Input, LSTM, Embedding, Dense, Dropout, Reshape
from keras.models import Model, Sequential
from keras.preprocessing import image
from keras.preprocessing.text import Tokenizer vision_model = Sequential()
vision_model.add(Conv3D(32, (3, 3, 3), activation='relu', padding='same', input_shape=(15, 28, 28, 3)))
vision_model.add(MaxPooling3D((2, 2, 2)))
# vision_model.add(Dropout(0.1))
vision_model.add(Conv3D(64, (3, 3, 3), activation='relu', padding='same'))
vision_model.add(MaxPooling3D((2, 2, 2)))
# vision_model.add(Dropout(0.1))
vision_model.add(Conv3D(128, (3, 3, 3), activation='relu', padding='same'))
vision_model.add(MaxPooling3D((2, 2, 2)))
# vision_model.add(Conv3D(256, (3, 3, 3), activation='relu', padding='same'))
# vision_model.add(MaxPooling3D((3, 3, 3)))
vision_model.add(Flatten()) image_input = Input(shape=(15, 28, 28, 3))
encoded_image = vision_model(image_input) question_input = Input(shape=(19,), dtype='int32')
embedded_question = Embedding(input_dim=10000, output_dim=256, input_length=19)(question_input)
encoded_question = LSTM(256)(embedded_question) merged = keras.layers.concatenate([encoded_image, encoded_question])
# output = Dense(500, activation='softmax')(merged)
output = Dense(7554, activation='softmax')(merged) vqa_model = Model(inputs=[image_input, question_input], outputs=output)
adam = keras.optimizers.adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
vqa_model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])
vqa_model.summary()
import pandas as pd
import numpy as np
import os
import random
import collections FRAMES_NUM = 15
max_len = 5 def toarray(str, maxlen):
arr = str.split(' ')
length = len(arr)
if (length < maxlen):
for _ in range(maxlen-length):
arr.append('$')
return arr def tovector(qqarr):
global max_len
qq_all_words = []
qq_all_words += ['$'] for itv in qqarr:
qq_all_words += [word for word in itv]
max_len = max(max_len, len(itv))
print("maxlen:",max_len)
qqcounter = collections.Counter(qq_all_words)
print("qqcounter:",len(qqcounter))
qq_counter_pairs = sorted(qqcounter.items(), key = lambda x : -x[1])
qqwords,_ = zip(*qq_counter_pairs) qq_word_num_map = dict(zip(qqwords, range(len(qqwords))))
qq_to_num = lambda word:qq_word_num_map.get(word, len(qqwords))
qq_vector = [list(map(qq_to_num, word)) for word in qqarr]
return qq_vector, qq_word_num_map, qq_to_num def tolabel(labels):
all_words = []
for itv in labels:
all_words.append([itv])
counter = collections.Counter(labels)
print("labelcounter:",len(counter))
counter_pairs = sorted(counter.items(), key = lambda x : -x[1])
words, _ = zip(*counter_pairs)
print(words)
print("wordslen:",len(words)) word_num_map = dict(zip(words, range(len(words))))
to_num = lambda word: word_num_map.get(word, len(words))
vector = [list(map(to_num, word)) for word in all_words]
return vector, word_num_map, to_num def randomsample(list, count, dicdir):
if len(list) > count:
sampleintlist = random.sample(range(len(list)), count)
else:
sampleintlist = []
for i in range(count):
sampleintlist.append(i % len(list))
sampleintlist.sort()
samplelist = []
for i in sampleintlist:
samplelist.append(os.path.join(dicdir, str(i) + ".jpg"))
return samplelist def getvideo(key):
dicdir = os.path.join(r"D:\ai\AIE04\tianchi\videoanswer\image", key)
list = os.listdir(dicdir)
samplelist = randomsample(list, FRAMES_NUM, dicdir)
return samplelist path = r"D:\ai\AIE04\VQADatasetA_20180815"
data_train = pd.read_csv(os.path.join(path, 'train.txt'), header=None) length= len(data_train)*FRAMES_NUM frames = []
qqarr = []
aaarr = []
qq = []
aa = []
labels = []
paths = []
for i in range(len(data_train)):
label, q1, a11, a12, a13, q2, a21, a22, a23, q3, a31, a32, a33, q4, a41, a42, a43, q5, a51, a52, a53 = data_train.loc[i]
print(label)
[paths.append(label) for j in range(15)] [qqarr.append(toarray(str(q1), 19)) for j in range(3)]
[qqarr.append(toarray(str(q2), 19)) for j in range(3)]
[qqarr.append(toarray(str(q3), 19)) for j in range(3)]
[qqarr.append(toarray(str(q4), 19)) for j in range(3)]
[qqarr.append(toarray(str(q5), 19)) for j in range(3)] labels.append(a11)
labels.append(a12)
labels.append(a13)
labels.append(a21)
labels.append(a22)
labels.append(a23)
labels.append(a31)
labels.append(a32)
labels.append(a33)
labels.append(a41)
labels.append(a42)
labels.append(a43)
labels.append(a51)
labels.append(a52)
labels.append(a53) qq_vector, qq_word_num_map, qq_to_num = tovector(qqarr) # print(labels)
vector, word_num_map, to_num = tolabel(labels)
# print(vector)
# print(word_num_map)
# print(to_num)
from nltk.probability import FreqDist
from collections import Counter
import train_data
from keras.preprocessing.text import Tokenizer
import numpy as np
from PIL import Image
from keras.utils import to_categorical
import math
import videomodel
from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint
import keras.backend as K
import keras
import cv2 def scheduler(epoch):
if epoch % 10 == 0 and epoch != 0:
lr = K.get_value(videomodel.vqa_model.optimizer.lr)
K.set_value(videomodel.vqa_model.optimizer.lr, lr * 0.9)
print("lr changed to {}".format(lr * 0.9))
return K.get_value(videomodel.vqa_model.optimizer.lr) def get_trainDataset(paths, question, answer, img_size):
num_samples = len(paths)
X = np.zeros((num_samples, 15, img_size, img_size, 3))
Q = np.array(question)
for i in range(num_samples):
# image_paths = frames[i]
image_paths = train_data.getvideo(paths[i])
# print("len:",len(image_paths))
for kk in range(len(image_paths)):
path = image_paths[kk]
# print(path)
img = Image.open(path)
img = img.resize((img_size, img_size))
# print(img)
X[i, kk, :, :, :] = np.array(img)
img.close()
Y = to_categorical(np.array(answer), 7554)
# print(X.shape)
# print(Q)
# print(Y)
return [X, Q], Y
def generate_for_train(paths, question, answer, img_size, batch_size):
while True:
# train_zip = list(zip(frames, question, answer))
# np.random.shuffle(train_zip)
# print(train_zip)
# frames, question, answer = zip(*train_zip)
k = len(paths)
epochs = math.ceil(k/batch_size)
for i in range(epochs):
s = i * batch_size
e = s + batch_size
if (e > k):
e = k
x, y = get_trainDataset(paths[s:e], question[s:e], answer[s:e], img_size)
yield (x, y) split = len(train_data.paths) - 6000
qsplit = split*15
print("split:", split)
batch_size = 10
reduce_lr = LearningRateScheduler(scheduler)
tensorboard = TensorBoard(log_dir='log', write_graph=True)
checkpoint = ModelCheckpoint("max.h5", monitor="val_acc", verbose=1, save_best_only="True", mode="auto")
h = videomodel.vqa_model.fit_generator(generate_for_train(train_data.paths[:split], train_data.qq_vector[:split], train_data.vector[:split], 28, batch_size),
steps_per_epoch=math.ceil(len(train_data.paths[0:split]) / batch_size), validation_data=generate_for_train(train_data.paths[split:], train_data.qq_vector[split:], train_data.vector[split:], 28, batch_size),
validation_steps=math.ceil(len(train_data.paths[split:]) / batch_size),
verbose=1, epochs=100, callbacks=[tensorboard, checkpoint])
计算图如下：
每段视频只取了15帧，每帧图片大小压缩到28*28，之所以这样是因为内存不够。即使这样在跑第二轮时也报了内存错误。看样子对个人来说，gpu（本人GTX1050）还不算是大的瓶颈（虽然一轮就需要5个小时），反而内存（本人8g内存，gpu2g）成了瓶颈。
下一篇文章再对此次参加的过程做下总结。