[쉬움주의] 마지막 8장 어텐션을 이용한 질의 응답

[유연수]

[쉬움주의] 마지막 8장 어텐션을 이용한 질의 응답, 코렙에서 완전 잘되는 코드

출처:

https://wikidocs.net/85470

질의응답_20200909_완전잘되는것.txt

!pip install customized_konlpy

from google.colab import drive

drive.mount('/content/gdrive')

from ckonlpy.tag import Twitter

from tensorflow.keras.utils import get_file

from tensorflow.keras.preprocessing.sequence import pad_sequences

from tensorflow.keras.utils import to_categorical

import numpy as np

from nltk import FreqDist

from functools import reduce

import os

import re

import matplotlib.pyplot as plt

#훈련 데이터 : https://bit.ly/31SqtHy

#테스트 데이터 : https://bit.ly/3f7rH5g

TRAIN_FILE = os.path.join("/content/gdrive/My Drive/Colab Notebooks/qa1_single-supporting-fact_train_kor.txt")

TEST_FILE = os.path.join("/content/gdrive/My Drive/Colab Notebooks/qa1_single-supporting-fact_test_kor.txt")

# conda install nltk

i = 0

lines = open(TRAIN_FILE , "rb")

for line in lines:

    line = line.decode("utf-8").strip()

    i = i + 1

    print(line)

    if i == 20:

      break

  

def read_data(dir):

    stories, questions, answers = [], [], [] # 각각 스토리, 질문, 답변을 저장할 예정

    story_temp = [] # 현재 시점의 스토리 임시 저장

    lines = open(dir, "rb")

    for line in lines:

        line = line.decode("utf-8") # b' 제거

        line = line.strip() # '\n' 제거

        idx, text = line.split(" ", 1) # 맨 앞에 있는 id number 분리

        # 여기까지는 모든 줄에 적용되는 전처리

        if int(idx) == 1:

            story_temp = []

        if "\t" in text: # 현재 읽는 줄이 질문 (tab) 답변 (tab)인 경우

            question, answer, _ = text.split("\t") # 질문과 답변을 각각 저장

            stories.append([x for x in story_temp if x]) # 지금까지의 누적 스토리를 스토리에 저장

            questions.append(question)

            answers.append(answer)

        else: # 현재 읽는 줄이 스토리인 경우

            story_temp.append(text) # 임시 저장

    lines.close()

    return stories, questions, answers    

#%%

train_data = read_data(TRAIN_FILE)

test_data = read_data(TEST_FILE)

#%%

train_stories, train_questions, train_answers = read_data(TRAIN_FILE)

test_stories, test_questions, test_answers = read_data(TEST_FILE)

print('훈련용 스토리의 개수 :', len(train_stories))

print('훈련용 질문의 개수 :',len(train_questions))

print('훈련용 답변의 개수 :',len(train_answers))

print('테스트용 스토리의 개수 :',len(test_stories))

print('테스트용 질문의 개수 :',len(test_questions))

print('테스트용 답변의 개수 :',len(test_answers))

#%%

print(train_stories[3572])

#%%

print(train_questions[3572])

#%%

print( train_answers[3572])

#%%

def tokenize(sent):

    return [ x.strip() for x in re.split('(\W+)?', sent) if x.strip()]

#%%

def preprocess_data(train_data, test_data):

    counter = FreqDist()

    # 두 문장의 story를 하나의 문장으로 통합하는 함수

    flatten = lambda data: reduce(lambda x, y: x + y, data)

    # 각 샘플의 길이를 저장하는 리스트

    story_len = []

    question_len = []

    for stories, questions, answers in [train_data, test_data]:

        for story in stories:

            stories = tokenize(flatten(story)) # 스토리의 문장들을 펼친 후 토큰화

            story_len.append(len(stories)) # 각 story의 길이 저장

            for word in stories: # 단어 집합에 단어 추가

                counter[word] += 1

        for question in questions:

            question = tokenize(question)

            question_len.append(len(question))

            for word in question:

                counter[word] += 1

        for answer in answers:

            answer = tokenize(answer)

            for word in answer:

                counter[word] += 1

    # 단어 집합 생성

    word2idx = {word : (idx + 1) for idx, (word, _) in enumerate(counter.most_common())}

    idx2word = {idx : word for word, idx in word2idx.items()}

    # 가장 긴 샘플의 길이

    story_max_len = np.max(story_len)

    question_max_len = np.max(question_len)

    return word2idx, idx2word, story_max_len, question_max_len

#%%

word2idx, idx2word, story_max_len, question_max_len = preprocess_data(train_data, test_data)

print(word2idx)

twitter = Twitter()

twitter.add_dictionary('은경이', 'Noun')

twitter.add_dictionary('경임이', 'Noun')

twitter.add_dictionary('수종이', 'Noun')

print(twitter.morphs('은경이는 화장실로 이동했습니다.'))

print(twitter.morphs('경임이는 정원으로 가버렸습니다.'))

print(twitter.morphs('수종이는 복도로 뛰어갔습니다.'))

print(twitter.morphs('필웅이는 부엌으로 복귀했습니다.'))

print(twitter.morphs('수종이는 사무실로 갔습니다.'))

print(twitter.morphs('은경이는 침실로 갔습니다.'))

def tokenize(sent):

    return twitter.morphs(sent)

word2idx, idx2word, story_max_len, question_max_len = preprocess_data(train_data, test_data)

print(word2idx)

vocab_size = len(word2idx) + 1

print(vocab_size)

print('스토리의 최대 길이 :',story_max_len)

print('질문의 최대 길이 :',question_max_len)

def vectorize(data, word2idx, story_maxlen, question_maxlen):

    Xs, Xq, Y = [], [], []

    flatten = lambda data: reduce(lambda x, y: x + y, data)

    stories, questions, answers = data

    for story, question, answer in zip(stories, questions, answers):

        xs = [word2idx[w] for w in tokenize(flatten(story))]

        xq = [word2idx[w] for w in tokenize(question)]

        Xs.append(xs)

        Xq.append(xq)

        Y.append(word2idx[answer])

        # 스토리와 질문은 각각의 최대 길이로 패딩

        # 정답은 원-핫 인코딩

    return pad_sequences(Xs, maxlen=story_maxlen),\

           pad_sequences(Xq, maxlen=question_maxlen),\

           to_categorical(Y, num_classes=len(word2idx) + 1)

Xstrain, Xqtrain, Ytrain = vectorize(train_data, word2idx, story_max_len, question_max_len)

Xstest, Xqtest, Ytest = vectorize(test_data, word2idx, story_max_len, question_max_len)

print(Xstrain.shape, Xqtrain.shape, Ytrain.shape, Xstest.shape, Xqtest.shape, Ytest.shape)

from tensorflow.keras.models import Sequential, Model

from tensorflow.keras.layers import Embedding

from tensorflow.keras.layers import Permute, dot, add, concatenate

from tensorflow.keras.layers import LSTM, Dense, Dropout, Input, Activation

# 에포크 횟수

train_epochs = 120

# 배치 크기

batch_size = 32

# 임베딩 크기

embed_size = 50

# LSTM의 크기

lstm_size = 64

# 과적합 방지 기법인 드롭아웃 적용 비율

dropout_rate = 0.30

# 플레이스 홀더. 입력을 담는 변수

input_sequence = Input((story_max_len,))

question = Input((question_max_len,))

print('Stories :', input_sequence)

print('Question:', question)

# 스토리를 위한 첫번째 임베딩. 그림에서의 Embedding A

input_encoder_m = Sequential()

input_encoder_m.add(Embedding(input_dim=vocab_size,

                              output_dim=embed_size))

input_encoder_m.add(Dropout(dropout_rate))

# 결과 : (samples, story_max_len, embedding_dim) / 샘플의 수, 문장의 최대 길이, 임베딩 벡터의 차원

# 스토리를 위한 두번째 임베딩. 그림에서의 Embedding C

# 임베딩 벡터의 차원을 question_max_len(질문의 최대 길이)로 한다.

input_encoder_c = Sequential()

input_encoder_c.add(Embedding(input_dim=vocab_size,

                              output_dim=question_max_len))

input_encoder_c.add(Dropout(dropout_rate))

# 결과 : (samples, story_max_len, question_max_len) / 샘플의 수, 문장의 최대 길이, 질문의 최대 길이(임베딩 벡터의 차원)

# 질문을 위한 임베딩. 그림에서의 Embedding B

question_encoder = Sequential()

question_encoder.add(Embedding(input_dim=vocab_size,

                               output_dim=embed_size,

                               input_length=question_max_len))

question_encoder.add(Dropout(dropout_rate))

# 결과 : (samples, question_max_len, embedding_dim) / 샘플의 수, 질문의 최대 길이, 임베딩 벡터의 차원

# 실질적인 임베딩 과정

input_encoded_m = input_encoder_m(input_sequence)

input_encoded_c = input_encoder_c(input_sequence)

question_encoded = question_encoder(question)

print('Input encoded m', input_encoded_m)

print('Input encoded c', input_encoded_c)

print('Question encoded', question_encoded)

# 스토리 단어들과 질문 단어들 간의 유사도를 구하는 과정

# 유사도는 내적을 사용한다.

match = dot([input_encoded_m, question_encoded], axes=-1, normalize=False)

match = Activation('softmax')(match)

print('Match shape', match)

# 결과 : (samples, story_maxlen, question_max_len) / 샘플의 수, 문장의 최대 길이, 질문의 최대 길이

# add the match matrix with the second input vector sequence

response = add([match, input_encoded_c])  # (samples, story_max_len, question_max_len)

response = Permute((2, 1))(response)  # (samples, question_max_len, story_max_len)

print('Response shape', response)

# concatenate the response vector with the question vector sequence

answer = concatenate([response, question_encoded])

print('Answer shape', answer)

answer = LSTM(lstm_size)(answer)  # Generate tensors of shape 32

answer = Dropout(dropout_rate)(answer)

answer = Dense(vocab_size)(answer)  # (samples, vocab_size)

# we output a probability distribution over the vocabulary

answer = Activation('softmax')(answer)

# build the final model

model = Model([input_sequence, question], answer)

model.compile(optimizer='rmsprop', loss='categorical_crossentropy',

              metrics=['acc'])

print(model.summary())

# start training the model

history = model.fit([Xstrain, Xqtrain],

         Ytrain, batch_size, train_epochs,

         validation_data=([Xstest, Xqtest], Ytest))

# save model

model.save('model.h5')

# plot accuracy and loss plot

plt.subplot(211)

plt.title("Accuracy")

plt.plot(history.history["acc"], color="g", label="train")

plt.plot(history.history["val_acc"], color="b", label="validation")

plt.legend(loc="best")

plt.subplot(212)

plt.title("Loss")

plt.plot(history.history["loss"], color="g", label="train")

plt.plot(history.history["val_loss"], color="b", label="validation")

plt.legend(loc="best")

plt.tight_layout()

plt.show()

# labels

ytest = np.argmax(Ytest, axis=1)

# get predictions

Ytest_ = model.predict([Xstest, Xqtest])

ytest_ = np.argmax(Ytest_, axis=1)

NUM_DISPLAY = 30

print("{:18}|{:5}|{}".format("질문", "실제값", "예측값"))

print(39 * "-")

for i in range(NUM_DISPLAY):

    question = " ".join([idx2word[x] for x in Xqtest[i].tolist()])

    label = idx2word[ytest[i]]

    prediction = idx2word[ytest_[i]]

    print("{:20}: {:7} {}".format(question, label, prediction))

poch 76/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0546 - acc: 0.9819 - val_loss: 0.1822 - val_acc: 0.9480
Epoch 77/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0529 - acc: 0.9838 - val_loss: 0.1895 - val_acc: 0.9500
Epoch 78/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0478 - acc: 0.9830 - val_loss: 0.2046 - val_acc: 0.9450
Epoch 79/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0447 - acc: 0.9854 - val_loss: 0.1777 - val_acc: 0.9520
Epoch 80/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0506 - acc: 0.9834 - val_loss: 0.1836 - val_acc: 0.9480
Epoch 81/120
313/313 [==============================] - 3s 10ms/step - loss: 0.0434 - acc: 0.9853 - val_loss: 0.1823 - val_acc: 0.9530
Epoch 82/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0390 - acc: 0.9865 - val_loss: 0.1898 - val_acc: 0.9520
Epoch 83/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0392 - acc: 0.9883 - val_loss: 0.1950 - val_acc: 0.9500
Epoch 84/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0430 - acc: 0.9860 - val_loss: 0.2025 - val_acc: 0.9500
Epoch 85/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0329 - acc: 0.9887 - val_loss: 0.1776 - val_acc: 0.9500
Epoch 86/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0319 - acc: 0.9892 - val_loss: 0.1613 - val_acc: 0.9520
Epoch 87/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0349 - acc: 0.9887 - val_loss: 0.1510 - val_acc: 0.9580
Epoch 88/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0373 - acc: 0.9890 - val_loss: 0.1632 - val_acc: 0.9540
Epoch 89/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0304 - acc: 0.9908 - val_loss: 0.1520 - val_acc: 0.9600
Epoch 90/120
313/313 [==============================] - 3s 10ms/step - loss: 0.0317 - acc: 0.9900 - val_loss: 0.1580 - val_acc: 0.9510
Epoch 91/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0267 - acc: 0.9900 - val_loss: 0.1615 - val_acc: 0.9550
Epoch 92/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0221 - acc: 0.9920 - val_loss: 0.1469 - val_acc: 0.9600
Epoch 93/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0284 - acc: 0.9915 - val_loss: 0.2101 - val_acc: 0.9510
Epoch 94/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0244 - acc: 0.9925 - val_loss: 0.1577 - val_acc: 0.9610
Epoch 95/120
313/313 [==============================] - 3s 10ms/step - loss: 0.0248 - acc: 0.9916 - val_loss: 0.1778 - val_acc: 0.9550
Epoch 96/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0226 - acc: 0.9932 - val_loss: 0.1531 - val_acc: 0.9610
Epoch 97/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0209 - acc: 0.9932 - val_loss: 0.1541 - val_acc: 0.9600
Epoch 98/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0249 - acc: 0.9918 - val_loss: 0.1766 - val_acc: 0.9620
Epoch 99/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0242 - acc: 0.9925 - val_loss: 0.1559 - val_acc: 0.9630
Epoch 100/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0175 - acc: 0.9940 - val_loss: 0.1458 - val_acc: 0.9560
Epoch 101/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0258 - acc: 0.9925 - val_loss: 0.1263 - val_acc: 0.9640
Epoch 102/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0187 - acc: 0.9943 - val_loss: 0.1757 - val_acc: 0.9560
Epoch 103/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0246 - acc: 0.9929 - val_loss: 0.1308 - val_acc: 0.9640
Epoch 104/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0224 - acc: 0.9942 - val_loss: 0.1673 - val_acc: 0.9600
Epoch 105/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0199 - acc: 0.9940 - val_loss: 0.1267 - val_acc: 0.9600
Epoch 106/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0165 - acc: 0.9943 - val_loss: 0.1345 - val_acc: 0.9650
Epoch 107/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0194 - acc: 0.9946 - val_loss: 0.1262 - val_acc: 0.9630
Epoch 108/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0189 - acc: 0.9941 - val_loss: 0.1270 - val_acc: 0.9650
Epoch 109/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0187 - acc: 0.9949 - val_loss: 0.1500 - val_acc: 0.9620
Epoch 110/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0235 - acc: 0.9940 - val_loss: 0.1155 - val_acc: 0.9650
Epoch 111/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0235 - acc: 0.9933 - val_loss: 0.1248 - val_acc: 0.9640
Epoch 112/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0140 - acc: 0.9957 - val_loss: 0.1634 - val_acc: 0.9650
Epoch 113/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0151 - acc: 0.9945 - val_loss: 0.1573 - val_acc: 0.9610
Epoch 114/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0211 - acc: 0.9940 - val_loss: 0.1488 - val_acc: 0.9610
Epoch 115/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0148 - acc: 0.9952 - val_loss: 0.1261 - val_acc: 0.9660
Epoch 116/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0165 - acc: 0.9948 - val_loss: 0.1388 - val_acc: 0.9660
Epoch 117/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0150 - acc: 0.9960 - val_loss: 0.1263 - val_acc: 0.9660
Epoch 118/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0129 - acc: 0.9962 - val_loss: 0.1311 - val_acc: 0.9690
Epoch 119/120
313/313 [==============================] - 3s 9ms/step - loss: 0.0200 - acc: 0.9950 - val_loss: 0.1113 - val_acc: 0.9710
Epoch 120/120
313/313 [==============================] - 3s 10ms/step - loss: 0.0165 - acc: 0.9960 - val_loss: 0.1383 - val_acc: 0.9640

질문                |실제값  |예측값
---------------------------------------
은경이 는 어디 야 ?        : 복도      복도
필웅이 는 어디 야 ?        : 화장실     화장실
경임이 는 어디 야 ?        : 부엌      부엌
경임이 는 어디 야 ?        : 복도      복도
경임이 는 어디 야 ?        : 부엌      부엌
경임이 는 어디 야 ?        : 복도      복도
경임이 는 어디 야 ?        : 정원      정원
수종이 는 어디 야 ?        : 복도      복도
경임이 는 어디 야 ?        : 사무실     사무실
수종이 는 어디 야 ?        : 사무실     사무실
필웅이 는 어디 야 ?        : 부엌      부엌
필웅이 는 어디 야 ?        : 정원      정원
수종이 는 어디 야 ?        : 사무실     사무실
필웅이 는 어디 야 ?        : 침실      침실
필웅이 는 어디 야 ?        : 침실      침실
은경이 는 어디 야 ?        : 부엌      부엌
은경이 는 어디 야 ?        : 정원      정원
은경이 는 어디 야 ?        : 부엌      부엌
수종이 는 어디 야 ?        : 사무실     사무실
은경이 는 어디 야 ?        : 부엌      부엌
필웅이 는 어디 야 ?        : 복도      복도
은경이 는 어디 야 ?        : 사무실     사무실
은경이 는 어디 야 ?        : 사무실     사무실
경임이 는 어디 야 ?        : 복도      복도
수종이 는 어디 야 ?        : 침실      침실
경임이 는 어디 야 ?        : 침실      침실
필웅이 는 어디 야 ?        : 침실      침실
수종이 는 어디 야 ?        : 부엌      부엌
수종이 는 어디 야 ?        : 부엌      부엌
수종이 는 어디 야 ?        : 부엌      사무실