diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 07cc71b..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0b679c --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Jupyter Notebook +.ipynb_checkpoints + +# VS Code +.vscode/ + +# MAC +.DS_Store + + +data/* \ No newline at end of file diff --git a/__pycache__/preprocession.cpython-37.pyc b/__pycache__/preprocession.cpython-37.pyc deleted file mode 100644 index 4816465..0000000 Binary files a/__pycache__/preprocession.cpython-37.pyc and /dev/null differ diff --git a/dataset.py b/dataset.py new file mode 100644 index 0000000..c678238 --- /dev/null +++ b/dataset.py @@ -0,0 +1,11 @@ +import json +from torch.utils.data import IterableDataset + +class ConceptFlowDataset(IterableDataset): + def __init__(self, txt_file, config): + self.root_dir = config.data_dir + self.txt_file = txt_file + + def __iter__(self): + f = open(f'{self.root_dir}/{self.txt_file}') + return map(json.loads, f) \ No newline at end of file diff --git a/model/.DS_Store b/model/.DS_Store deleted file mode 100644 index 5ec44c2..0000000 Binary files a/model/.DS_Store and /dev/null differ diff --git a/model/__pycache__/__init__.cpython-37.pyc b/model/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index fe739dd..0000000 Binary files a/model/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/model/__pycache__/central.cpython-37.pyc b/model/__pycache__/central.cpython-37.pyc deleted file mode 100644 index 48c83bb..0000000 Binary files a/model/__pycache__/central.cpython-37.pyc and /dev/null differ diff --git a/model/__pycache__/conceptflow.cpython-37.pyc b/model/__pycache__/conceptflow.cpython-37.pyc deleted file mode 100644 index e8b3893..0000000 Binary files a/model/__pycache__/conceptflow.cpython-37.pyc and /dev/null differ diff --git a/model/__pycache__/embedding.cpython-37.pyc b/model/__pycache__/embedding.cpython-37.pyc deleted file mode 100644 index 2d0f57b..0000000 Binary files a/model/__pycache__/embedding.cpython-37.pyc and /dev/null differ diff --git a/model/__pycache__/model.cpython-37.pyc b/model/__pycache__/model.cpython-37.pyc deleted file mode 100644 index 838e5f1..0000000 Binary files a/model/__pycache__/model.cpython-37.pyc and /dev/null differ diff --git a/model/__pycache__/outer.cpython-37.pyc b/model/__pycache__/outer.cpython-37.pyc deleted file mode 100644 index 5f924ef..0000000 Binary files a/model/__pycache__/outer.cpython-37.pyc and /dev/null differ diff --git a/model/central.py b/model/central.py index 847aac3..af8d1da 100644 --- a/model/central.py +++ b/model/central.py @@ -134,18 +134,14 @@ class LeftMMFixed(torch.autograd.Function): Implementation of matrix multiplication of a Sparse Variable with a Dense Variable, returning a Dense one. This is added because there's no autograd for sparse yet. No gradient computed on the sparse weights. """ - - def __init__(self): - super(LeftMMFixed, self).__init__() - self.sparse_weights = None - - def forward(self, sparse_weights, x): - if self.sparse_weights is None: - self.sparse_weights = sparse_weights - return torch.mm(self.sparse_weights, x) - - def backward(self, grad_output): - sparse_weights = self.sparse_weights + @staticmethod + def forward(ctx, sparse_weights, x): + ctx.sparse_weights = sparse_weights + return torch.mm(ctx.sparse_weights, x) + + @staticmethod + def backward(ctx, grad_output): + sparse_weights = ctx.sparse_weights return None, torch.mm(sparse_weights.t(), grad_output) I = X._indices() @@ -156,6 +152,5 @@ def backward(self, grad_output): lookup = Y[I[0, :], I[2, :], :] X_I = torch.stack((I[0, :] * M + I[1, :], use_cuda(torch.arange(Z).type(torch.LongTensor))), 0) S = use_cuda(Variable(torch.sparse.FloatTensor(X_I, V, torch.Size([B * M, Z])), requires_grad=False)) - prod_op = LeftMMFixed() - prod = prod_op(S, lookup) + prod = LeftMMFixed.apply(S, lookup) return prod.view(B, M, K) diff --git a/preprocession.py b/preprocession.py index 8d65461..02c97ec 100644 --- a/preprocession.py +++ b/preprocession.py @@ -3,8 +3,10 @@ import json import torch from utils import padding, padding_triple_id, build_kb_adj_mat +from dataset import ConceptFlowDataset +from torch.utils.data import DataLoader + - def prepare_data(config): global csk_entities, csk_triples, kb_dict, dict_csk_entities, dict_csk_triples @@ -17,22 +19,9 @@ def prepare_data(config): kb_dict = d['dict_csk'] dict_csk_entities = d['dict_csk_entities'] dict_csk_triples = d['dict_csk_triples'] - - data_train, data_test = [], [] - - if config.is_train: - with open('%s/trainset.txt' % config.data_dir) as f: - for idx, line in enumerate(f): - if idx % 100000 == 0: print('read train file line %d' % idx) - data_train.append(json.loads(line)) + return raw_vocab - with open('%s/testset.txt' % config.data_dir) as f: - for line in f: - data_test.append(json.loads(line)) - - return raw_vocab, data_train, data_test - def build_vocab(path, raw_vocab, config, trans='transE'): print("Creating word vocabulary...") @@ -103,180 +92,198 @@ def build_vocab(path, raw_vocab, config, trans='transE'): return word2id, entity2id, vocab_list, embed, entity_list, entity_embed, relation_list, relation_embed, entity_relation_embed -def gen_batched_data(data, config, word2id, entity2id): - global csk_entities, csk_triples, kb_dict, dict_csk_entities, dict_csk_triples - encoder_len = max([len(item['post']) for item in data])+1 - - decoder_len = max([len(item['response']) for item in data])+1 - triple_num = max([len(item['all_triples_one_hop']) for item in data]) - entity_len = max([len(item['all_entities_one_hop']) + max(item['post_triples']) for item in data]) - only_two_entity_len = max([len(item['only_two']) for item in data]) - triple_num_one_two = max([len(item['one_two_triple']) for item in data]) - triple_len_one_two = max([len(tri) for item in data for tri in item['one_two_triple']]) - posts_id = np.full((len(data), encoder_len), 0, dtype=int) - responses_id = np.full((len(data), decoder_len), 0, dtype=int) - responses_length = [] - # posts_length = [] - local_entity_length = [] - only_two_entity_length = [] - local_entity = [] - only_two_entity = [] - kb_fact_rels = np.full((len(data), triple_num), 2, dtype=int) - kb_adj_mats = np.empty(len(data), dtype=object) - q2e_adj_mats = np.full((len(data), entity_len), 0, dtype=int) - match_entity_one_hop = np.full((len(data), decoder_len), -1, dtype=int) - match_entity_only_two = np.full((len(data), decoder_len), -1, dtype=int) - one_two_triples_id = [] - g2l_only_two_list = [] - # o2t_entity_index_list = [] - - next_id = 0 - for item in data: - # posts - for i, post_word in enumerate(padding(item['post'], encoder_len)): - if post_word in word2id: - posts_id[next_id, i] = word2id[post_word] - - else: - posts_id[next_id, i] = word2id['_UNK'] +def get_data(config, word2id, entity2id): + collate = _build_collate(config, word2id, entity2id) + + train_loader = None + if config.is_train: + train_dataset = ConceptFlowDataset('trainset.txt',config) + train_loader = DataLoader(train_dataset,batch_size=config.batch_size,collate_fn=collate) + + test_dataset = ConceptFlowDataset('testset.txt',config) + test_loader = DataLoader(test_dataset,batch_size=config.batch_size,collate_fn=collate) + + return train_loader,test_loader + +def _build_collate(config, word2id, entity2id): + + def gen_batched_data(data): + global csk_entities, csk_triples, kb_dict, dict_csk_entities, dict_csk_triples + + encoder_len = max([len(item['post']) for item in data])+1 + + decoder_len = max([len(item['response']) for item in data])+1 + triple_num = max([len(item['all_triples_one_hop']) for item in data]) + entity_len = max([len(item['all_entities_one_hop']) + max(item['post_triples']) for item in data]) + only_two_entity_len = max([len(item['only_two']) for item in data]) + triple_num_one_two = max([len(item['one_two_triple']) for item in data]) + triple_len_one_two = max([len(tri) for item in data for tri in item['one_two_triple']]) + posts_id = np.full((len(data), encoder_len), 0, dtype=int) + responses_id = np.full((len(data), decoder_len), 0, dtype=int) + responses_length = [] + # posts_length = [] + local_entity_length = [] + only_two_entity_length = [] + local_entity = [] + only_two_entity = [] + kb_fact_rels = np.full((len(data), triple_num), 2, dtype=int) + kb_adj_mats = np.empty(len(data), dtype=object) + q2e_adj_mats = np.full((len(data), entity_len), 0, dtype=int) + match_entity_one_hop = np.full((len(data), decoder_len), -1, dtype=int) + match_entity_only_two = np.full((len(data), decoder_len), -1, dtype=int) + one_two_triples_id = [] + g2l_only_two_list = [] + # o2t_entity_index_list = [] + + next_id = 0 + for item in data: + # posts + for i, post_word in enumerate(padding(item['post'], encoder_len)): + if post_word in word2id: + posts_id[next_id, i] = word2id[post_word] + + else: + posts_id[next_id, i] = word2id['_UNK'] + + # responses + for i, response_word in enumerate(padding(item['response'], decoder_len)): + if response_word in word2id: + responses_id[next_id, i] = word2id[response_word] + + else: + responses_id[next_id, i] = word2id['_UNK'] + + # responses_length + responses_length.append(len(item['response']) + 1) - # responses - for i, response_word in enumerate(padding(item['response'], decoder_len)): - if response_word in word2id: - responses_id[next_id, i] = word2id[response_word] + # local_entity + local_entity_tmp = [] + for i in range(len(item['post_triples'])): + if item['post_triples'][i] == 0: + continue + elif item['post'][i] not in entity2id: + continue + elif entity2id[item['post'][i]] in local_entity_tmp: + continue + else: + local_entity_tmp.append(entity2id[item['post'][i]]) + + for entity_index in item['all_entities_one_hop']: + if csk_entities[entity_index] not in entity2id: + continue + if entity2id[csk_entities[entity_index]] in local_entity_tmp: + continue + else: + local_entity_tmp.append(entity2id[csk_entities[entity_index]]) + local_entity_len_tmp = len(local_entity_tmp) + local_entity_tmp += [1] * (entity_len - len(local_entity_tmp)) + local_entity.append(local_entity_tmp) + + # kb_adj_mat and kb_fact_rel + g2l = dict() + for i in range(len(local_entity_tmp)): + g2l[local_entity_tmp[i]] = i + + entity2fact_e, entity2fact_f = [], [] + fact2entity_f, fact2entity_e = [], [] + + tmp_count = 0 + for i in range(len(item['all_triples_one_hop'])): + sbj = csk_triples[item['all_triples_one_hop'][i]].split()[0][:-1] + rel = csk_triples[item['all_triples_one_hop'][i]].split()[1][:-1] + obj = csk_triples[item['all_triples_one_hop'][i]].split()[2] + + if (sbj not in entity2id) or (obj not in entity2id): + continue + if (entity2id[sbj] not in g2l) or (entity2id[obj] not in g2l): + continue - else: - responses_id[next_id, i] = word2id['_UNK'] - - # responses_length - responses_length.append(len(item['response']) + 1) - - # local_entity - local_entity_tmp = [] - for i in range(len(item['post_triples'])): - if item['post_triples'][i] == 0: - continue - elif item['post'][i] not in entity2id: - continue - elif entity2id[item['post'][i]] in local_entity_tmp: - continue - else: - local_entity_tmp.append(entity2id[item['post'][i]]) - - for entity_index in item['all_entities_one_hop']: - if csk_entities[entity_index] not in entity2id: - continue - if entity2id[csk_entities[entity_index]] in local_entity_tmp: - continue - else: - local_entity_tmp.append(entity2id[csk_entities[entity_index]]) - local_entity_len_tmp = len(local_entity_tmp) - local_entity_tmp += [1] * (entity_len - len(local_entity_tmp)) - local_entity.append(local_entity_tmp) - - # kb_adj_mat and kb_fact_rel - g2l = dict() - for i in range(len(local_entity_tmp)): - g2l[local_entity_tmp[i]] = i - - entity2fact_e, entity2fact_f = [], [] - fact2entity_f, fact2entity_e = [], [] - - tmp_count = 0 - for i in range(len(item['all_triples_one_hop'])): - sbj = csk_triples[item['all_triples_one_hop'][i]].split()[0][:-1] - rel = csk_triples[item['all_triples_one_hop'][i]].split()[1][:-1] - obj = csk_triples[item['all_triples_one_hop'][i]].split()[2] - - if (sbj not in entity2id) or (obj not in entity2id): - continue - if (entity2id[sbj] not in g2l) or (entity2id[obj] not in g2l): - continue + entity2fact_e += [g2l[entity2id[sbj]]] + entity2fact_f += [tmp_count] + fact2entity_f += [tmp_count] + fact2entity_e += [g2l[entity2id[obj]]] + kb_fact_rels[next_id, tmp_count] = entity2id[rel] + tmp_count += 1 + + kb_adj_mats[next_id] = (np.array(entity2fact_f, dtype=int), np.array(entity2fact_e, dtype=int), np.array([1.0] * len(entity2fact_f))), (np.array(fact2entity_e, dtype=int), np.array(fact2entity_f, dtype=int), np.array([1.0] * len(fact2entity_e))) - entity2fact_e += [g2l[entity2id[sbj]]] - entity2fact_f += [tmp_count] - fact2entity_f += [tmp_count] - fact2entity_e += [g2l[entity2id[obj]]] - kb_fact_rels[next_id, tmp_count] = entity2id[rel] - tmp_count += 1 - - kb_adj_mats[next_id] = (np.array(entity2fact_f, dtype=int), np.array(entity2fact_e, dtype=int), np.array([1.0] * len(entity2fact_f))), (np.array(fact2entity_e, dtype=int), np.array(fact2entity_f, dtype=int), np.array([1.0] * len(fact2entity_e))) - - # q2e_adj_mat - for i in range(len(item['post_triples'])): - if item['post_triples'][i] == 0: - continue - elif item['post'][i] not in entity2id: - continue - else: - q2e_adj_mats[next_id, g2l[entity2id[item['post'][i]]]] = 1 - - # match_entity_one_hop - for i in range(len(item['match_response_index_one_hop'])): - if item['match_response_index_one_hop'][i] == -1: - continue - if csk_entities[item['match_response_index_one_hop'][i]] not in entity2id: - continue - if entity2id[csk_entities[item['match_response_index_one_hop'][i]]] not in g2l: - continue - else: - match_entity_one_hop[next_id, i] = g2l[entity2id[csk_entities[item['match_response_index_one_hop'][i]]]] - - # only_two_entity - only_two_entity_tmp = [] - for entity_index in item['only_two']: - if csk_entities[entity_index] not in entity2id: - continue - if entity2id[csk_entities[entity_index]] in only_two_entity_tmp: - continue - else: - only_two_entity_tmp.append(entity2id[csk_entities[entity_index]]) - only_two_entity_len_tmp = len(only_two_entity_tmp) - only_two_entity_tmp += [1] * (only_two_entity_len - len(only_two_entity_tmp)) - only_two_entity.append(only_two_entity_tmp) - - # match_entity_two_hop - g2l_only_two = dict() - for i in range(len(only_two_entity_tmp)): - g2l_only_two[only_two_entity_tmp[i]] = i - - for i in range(len(item['match_response_index_only_two'])): - if item['match_response_index_only_two'][i] == -1: - continue - if csk_entities[item['match_response_index_only_two'][i]] not in entity2id: - continue - else: - match_entity_only_two[next_id, i] = g2l_only_two[entity2id[csk_entities[item['match_response_index_only_two'][i]]]] - - # one_two_triple - one_two_triples_id.append(padding_triple_id(entity2id, [[csk_triples[x].split(', ') for x in triple] for triple in item['one_two_triple']], triple_num_one_two, triple_len_one_two)) + # q2e_adj_mat + for i in range(len(item['post_triples'])): + if item['post_triples'][i] == 0: + continue + elif item['post'][i] not in entity2id: + continue + else: + q2e_adj_mats[next_id, g2l[entity2id[item['post'][i]]]] = 1 + + # match_entity_one_hop + for i in range(len(item['match_response_index_one_hop'])): + if item['match_response_index_one_hop'][i] == -1: + continue + if csk_entities[item['match_response_index_one_hop'][i]] not in entity2id: + continue + if entity2id[csk_entities[item['match_response_index_one_hop'][i]]] not in g2l: + continue + else: + match_entity_one_hop[next_id, i] = g2l[entity2id[csk_entities[item['match_response_index_one_hop'][i]]]] + + # only_two_entity + only_two_entity_tmp = [] + for entity_index in item['only_two']: + if csk_entities[entity_index] not in entity2id: + continue + if entity2id[csk_entities[entity_index]] in only_two_entity_tmp: + continue + else: + only_two_entity_tmp.append(entity2id[csk_entities[entity_index]]) + only_two_entity_len_tmp = len(only_two_entity_tmp) + only_two_entity_tmp += [1] * (only_two_entity_len - len(only_two_entity_tmp)) + only_two_entity.append(only_two_entity_tmp) + + # match_entity_two_hop + g2l_only_two = dict() + for i in range(len(only_two_entity_tmp)): + g2l_only_two[only_two_entity_tmp[i]] = i + + for i in range(len(item['match_response_index_only_two'])): + if item['match_response_index_only_two'][i] == -1: + continue + if csk_entities[item['match_response_index_only_two'][i]] not in entity2id: + continue + else: + match_entity_only_two[next_id, i] = g2l_only_two[entity2id[csk_entities[item['match_response_index_only_two'][i]]]] + + # one_two_triple + one_two_triples_id.append(padding_triple_id(entity2id, [[csk_triples[x].split(', ') for x in triple] for triple in item['one_two_triple']], triple_num_one_two, triple_len_one_two)) + + ############################ g2l_only_two + g2l_only_two_list.append(g2l_only_two) + + # local_entity_length + local_entity_length.append(local_entity_len_tmp) + + # only_two_entity_length + only_two_entity_length.append(only_two_entity_len_tmp) + + next_id += 1 + + batched_data = {'query_text': np.array(posts_id), + 'answer_text': np.array(responses_id), + 'local_entity': np.array(local_entity), + 'responses_length': responses_length, + 'q2e_adj_mat': np.array(q2e_adj_mats), + 'kb_adj_mat': build_kb_adj_mat(kb_adj_mats, config.fact_dropout), + 'kb_fact_rel': np.array(kb_fact_rels), + 'match_entity_one_hop': np.array(match_entity_one_hop), + 'only_two_entity': np.array(only_two_entity), + 'match_entity_only_two': np.array(match_entity_only_two), + 'one_two_triples_id': np.array(one_two_triples_id), + 'word2id': word2id, + 'entity2id': entity2id, + 'local_entity_length': local_entity_length, + 'only_two_entity_length': only_two_entity_length} - ############################ g2l_only_two - g2l_only_two_list.append(g2l_only_two) - - # local_entity_length - local_entity_length.append(local_entity_len_tmp) - - # only_two_entity_length - only_two_entity_length.append(only_two_entity_len_tmp) - - next_id += 1 - - batched_data = {'query_text': np.array(posts_id), - 'answer_text': np.array(responses_id), - 'local_entity': np.array(local_entity), - 'responses_length': responses_length, - 'q2e_adj_mat': np.array(q2e_adj_mats), - 'kb_adj_mat': build_kb_adj_mat(kb_adj_mats, config.fact_dropout), - 'kb_fact_rel': np.array(kb_fact_rels), - 'match_entity_one_hop': np.array(match_entity_one_hop), - 'only_two_entity': np.array(only_two_entity), - 'match_entity_only_two': np.array(match_entity_only_two), - 'one_two_triples_id': np.array(one_two_triples_id), - 'word2id': word2id, - 'entity2id': entity2id, - 'local_entity_length': local_entity_length, - 'only_two_entity_length': only_two_entity_length} - - return batched_data + return batched_data + + return gen_batched_data diff --git a/requirements.txt b/requirements.txt index 1819cd6..f07e0e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,7 +63,7 @@ PySocks==1.7.1 python-dateutil==2.8.1 pytorch-pretrained-bert==0.6.2 pytz==2019.3 -PyYAML==5.3.1 +PyYAML==5.4 regex==2020.2.20 requests==2.23.0 s3transfer==0.3.3 diff --git a/split_trainset.py b/split_trainset.py new file mode 100644 index 0000000..ddf045b --- /dev/null +++ b/split_trainset.py @@ -0,0 +1,15 @@ +import os + +def to_gb(b): + return round(b/float(1<<30),1) + +max_gb = 1 + +with open(f'./data/trainset_{max_gb}gb.txt','w') as f1: + with open('./data/trainset.txt','r') as f2: + for line in f2: + f1.write(line) + if to_gb(os.fstat(f1.fileno()).st_size) >= max_gb: + break + + \ No newline at end of file diff --git a/train.py b/train.py index 92f3444..4a9c7f9 100644 --- a/train.py +++ b/train.py @@ -1,171 +1,168 @@ -#coding:utf-8 -import numpy as np -import json -from model import ConceptFlow, use_cuda -from preprocession import prepare_data, build_vocab, gen_batched_data -import torch -import warnings -import yaml -import os -warnings.filterwarnings('ignore') - -csk_triples, csk_entities, kb_dict = [], [], [] -dict_csk_entities, dict_csk_triples = {}, {} -class Config(): - def __init__(self, path): - self.config_path = path - self._get_config() - - def _get_config(self): - with open(self.config_path, "r") as setting: - config = yaml.load(setting) - self.is_train = config['is_train'] - self.test_model_path = config['test_model_path'] - self.embed_units = config['embed_units'] - self.symbols = config['symbols'] - self.units = config['units'] - self.layers = config['layers'] - self.batch_size = config['batch_size'] - self.data_dir = config['data_dir'] - self.num_epoch = config['num_epoch'] - self.lr_rate = config['lr_rate'] - self.lstm_dropout = config['lstm_dropout'] - self.linear_dropout = config['linear_dropout'] - self.max_gradient_norm = config['max_gradient_norm'] - self.trans_units = config['trans_units'] - self.gnn_layers = config['gnn_layers'] - self.fact_dropout = config['fact_dropout'] - self.fact_scale = config['fact_scale'] - self.pagerank_lambda = config['pagerank_lambda'] - self.result_dir_name = config['result_dir_name'] - - def list_all_member(self): - for name, value in vars(self).items(): - print('%s = %s' % (name, value)) - - -def run(model, data_train, config, word2id, entity2id): - batched_data = gen_batched_data(data_train, config, word2id, entity2id) - - if model.is_inference == True: - word_index, selector = model(batched_data) - return word_index, selector - else: - decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, only_two_neg_num = model(batched_data) - return decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, only_two_neg_num - -def train(config, model, data_train, data_test, word2id, entity2id, model_optimizer): - for epoch in range(config.num_epoch): - print ("epoch: ", epoch) - sentence_ppx_loss = 0 - sentence_ppx_word_loss = 0 - sentence_ppx_local_loss = 0 - sentence_ppx_only_two_loss = 0 - - word_cut = use_cuda(torch.Tensor([0])) - local_cut = use_cuda(torch.Tensor([0])) - only_two_cut = use_cuda(torch.Tensor([0])) - - count = 0 - for iteration in range(len(data_train) // config.batch_size): - decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, \ - only_two_neg_num = run(model, data_train[(iteration * config.batch_size):(iteration * \ - config.batch_size + config.batch_size)], config, word2id, entity2id) - sentence_ppx_loss += torch.sum(sentence_ppx).data - sentence_ppx_word_loss += torch.sum(sentence_ppx_word).data - sentence_ppx_local_loss += torch.sum(sentence_ppx_local).data - sentence_ppx_only_two_loss += torch.sum(sentence_ppx_only_two).data - - word_cut += word_neg_num - local_cut += local_neg_num - only_two_cut += only_two_neg_num - - model_optimizer.zero_grad() - decoder_loss.backward() - torch.nn.utils.clip_grad_norm(model.parameters(), config.max_gradient_norm) - model_optimizer.step() - - if count % 50 == 0: - print ("iteration:", iteration, "Loss:", decoder_loss.data) - count += 1 - - print ("perplexity for epoch", epoch + 1, ":", np.exp(sentence_ppx_loss.cpu() / len(data_train)), " ppx_word: ", \ - np.exp(sentence_ppx_word_loss.cpu() / (len(data_train) - int(word_cut))), " ppx_local: ", \ - np.exp(sentence_ppx_local_loss.cpu() / (len(data_train) - int(local_cut))), " ppx_only_two: ", \ - np.exp(sentence_ppx_only_two_loss.cpu() / (len(data_train) - int(only_two_cut)))) - - torch.save(model.state_dict(), config.result_dir_name + '/' + '_epoch_' + str(epoch + 1) + '.pkl') - ppx, ppx_word, ppx_local, ppx_only_two = evaluate(model, data_test, config, word2id, entity2id, epoch + 1) - ppx_f = open(config.result_dir_name + '/result.txt','a') - ppx_f.write("epoch " + str(epoch + 1) + " ppx: " + str(ppx) + " ppx_word: " + str(ppx_word) + " ppx_local: " + \ - str(ppx_local) + " ppx_only_two: " + str(ppx_only_two) + '\n') - ppx_f.close() - -def evaluate(model, data_test, config, word2id, entity2id, epoch = 0, model_path = None): - if model_path != None: - model.load_state_dict(torch.load(model_path)) - sentence_ppx_loss = 0 - sentence_ppx_word_loss = 0 - sentence_ppx_local_loss = 0 - sentence_ppx_only_two_loss = 0 - word_cut = use_cuda(torch.Tensor([0])) - local_cut = use_cuda(torch.Tensor([0])) - only_two_cut = use_cuda(torch.Tensor([0])) - count = 0 - id2word = dict() - for key in word2id.keys(): - id2word[word2id[key]] = key - - - for iteration in range(len(data_test) // config.batch_size): - - decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, \ - local_neg_num, only_two_neg_num = run(model, data_test[(iteration * config.batch_size):(iteration * \ - config.batch_size + config.batch_size)], config, word2id, entity2id) - sentence_ppx_loss += torch.sum(sentence_ppx).data - sentence_ppx_word_loss += torch.sum(sentence_ppx_word).data - sentence_ppx_local_loss += torch.sum(sentence_ppx_local).data - sentence_ppx_only_two_loss += torch.sum(sentence_ppx_only_two).data - - word_cut += word_neg_num - local_cut += local_neg_num - only_two_cut += only_two_neg_num - - if count % 50 == 0: - print ("iteration for evaluate:", iteration, "Loss:", decoder_loss.data) - count += 1 - - model.is_inference = False - if model_path != None: - print(' perplexity on test set:', np.exp(sentence_ppx_loss.cpu() / len(data_test)), \ - np.exp(sentence_ppx_word_loss.cpu() / (len(data_test) - int(word_cut))), np.exp(sentence_ppx_local_loss.cpu() / (len(data_test) \ - - int(local_cut))), np.exp(sentence_ppx_only_two_loss.cpu() / (len(data_test) - int(only_two_cut)))) - exit() - print(' perplexity on test set:', np.exp(sentence_ppx_loss.cpu() / len(data_test)), np.exp(sentence_ppx_word_loss.cpu() / \ - (len(data_test) - int(word_cut))), np.exp(sentence_ppx_local_loss.cpu() / (len(data_test) - int(local_cut))), \ - np.exp(sentence_ppx_only_two_loss.cpu() / (len(data_test) - int(only_two_cut)))) - return np.exp(sentence_ppx_loss.cpu() / len(data_test)), np.exp(sentence_ppx_word_loss.cpu() / (len(data_test) - int(word_cut))), \ - np.exp(sentence_ppx_local_loss.cpu() / (len(data_test) - int(local_cut))), np.exp(sentence_ppx_only_two_loss.cpu() / \ - (len(data_test) - int(only_two_cut))) - -def main(): - config = Config('config.yml') - config.list_all_member() - raw_vocab, data_train, data_test = prepare_data(config) - word2id, entity2id, vocab, embed, entity_vocab, entity_embed, relation_vocab, relation_embed, entity_relation_embed = build_vocab(config.data_dir, raw_vocab, config = config) - model = use_cuda(ConceptFlow(config, embed, entity_relation_embed)) - - model_optimizer = torch.optim.Adam(model.parameters(), lr = config.lr_rate) - - if not os.path.exists(config.result_dir_name): - os.mkdir(config.result_dir_name) - ppx_f = open(config.result_dir_name + '/result.txt','a') - for name, value in vars(config).items(): - ppx_f.write('%s = %s' % (name, value) + '\n') - - if config.is_train == False: - evaluate(model, data_test, config, word2id, entity2id, 0, model_path = config.test_model_path) - exit() - train(config, model, data_train, data_test, word2id, entity2id, model_optimizer) - -main() +#coding:utf-8 +import numpy as np +import json +from model import ConceptFlow, use_cuda +from preprocession import prepare_data, build_vocab, get_data +import torch +import warnings +import yaml +import os +warnings.filterwarnings('ignore') + +csk_triples, csk_entities, kb_dict = [], [], [] +dict_csk_entities, dict_csk_triples = {}, {} +class Config(): + def __init__(self, path): + self.config_path = path + self._get_config() + + def _get_config(self): + with open(self.config_path, "r") as setting: + config = yaml.load(setting) + self.is_train = config['is_train'] + self.test_model_path = config['test_model_path'] + self.embed_units = config['embed_units'] + self.symbols = config['symbols'] + self.units = config['units'] + self.layers = config['layers'] + self.batch_size = config['batch_size'] + self.data_dir = config['data_dir'] + self.num_epoch = config['num_epoch'] + self.lr_rate = config['lr_rate'] + self.lstm_dropout = config['lstm_dropout'] + self.linear_dropout = config['linear_dropout'] + self.max_gradient_norm = config['max_gradient_norm'] + self.trans_units = config['trans_units'] + self.gnn_layers = config['gnn_layers'] + self.fact_dropout = config['fact_dropout'] + self.fact_scale = config['fact_scale'] + self.pagerank_lambda = config['pagerank_lambda'] + self.result_dir_name = config['result_dir_name'] + + def list_all_member(self): + for name, value in vars(self).items(): + print('%s = %s' % (name, value)) + + +def run(model, batched_data): + if model.is_inference == True: + word_index, selector = model(batched_data) + return word_index, selector + else: + decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, only_two_neg_num = model(batched_data) + return decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, only_two_neg_num + +def train(config, model, data_train, data_test, word2id, entity2id, model_optimizer): + for epoch in range(config.num_epoch): + print ("epoch: ", epoch) + sentence_ppx_loss = 0 + sentence_ppx_word_loss = 0 + sentence_ppx_local_loss = 0 + sentence_ppx_only_two_loss = 0 + + word_cut = use_cuda(torch.Tensor([0])) + local_cut = use_cuda(torch.Tensor([0])) + only_two_cut = use_cuda(torch.Tensor([0])) + + data_len = 0 + for iteration, batch in enumerate(data_train): + decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, local_neg_num, \ + only_two_neg_num = run(model, batch) + sentence_ppx_loss += torch.sum(sentence_ppx).data + sentence_ppx_word_loss += torch.sum(sentence_ppx_word).data + sentence_ppx_local_loss += torch.sum(sentence_ppx_local).data + sentence_ppx_only_two_loss += torch.sum(sentence_ppx_only_two).data + + word_cut += word_neg_num + local_cut += local_neg_num + only_two_cut += only_two_neg_num + + model_optimizer.zero_grad() + decoder_loss.backward() + torch.nn.utils.clip_grad_norm(model.parameters(), config.max_gradient_norm) + model_optimizer.step() + + if iteration % 50 == 0: + print ("iteration:", iteration, "Loss:", decoder_loss.data) + data_len += len(batch['query_text']) + + print ("perplexity for epoch", epoch + 1, ":", np.exp(sentence_ppx_loss.cpu() / data_len), " ppx_word: ", \ + np.exp(sentence_ppx_word_loss.cpu() / (data_len - int(word_cut))), " ppx_local: ", \ + np.exp(sentence_ppx_local_loss.cpu() / (data_len - int(local_cut))), " ppx_only_two: ", \ + np.exp(sentence_ppx_only_two_loss.cpu() / (data_len - int(only_two_cut)))) + + torch.save(model.state_dict(), config.result_dir_name + '/' + '_epoch_' + str(epoch + 1) + '.pkl') + ppx, ppx_word, ppx_local, ppx_only_two = evaluate(model, data_test, config, word2id, entity2id, epoch + 1) + ppx_f = open(config.result_dir_name + '/result.txt','a') + ppx_f.write("epoch " + str(epoch + 1) + " ppx: " + str(ppx) + " ppx_word: " + str(ppx_word) + " ppx_local: " + \ + str(ppx_local) + " ppx_only_two: " + str(ppx_only_two) + '\n') + ppx_f.close() + +def evaluate(model, data_test, config, word2id, entity2id, epoch = 0, model_path = None): + if model_path != None: + model.load_state_dict(torch.load(model_path)) + sentence_ppx_loss = 0 + sentence_ppx_word_loss = 0 + sentence_ppx_local_loss = 0 + sentence_ppx_only_two_loss = 0 + word_cut = use_cuda(torch.Tensor([0])) + local_cut = use_cuda(torch.Tensor([0])) + only_two_cut = use_cuda(torch.Tensor([0])) + id2word = dict() + for key in word2id.keys(): + id2word[word2id[key]] = key + + data_len = 0 + for iteration, batch in enumerate(data_test): + + decoder_loss, sentence_ppx, sentence_ppx_word, sentence_ppx_local, sentence_ppx_only_two, word_neg_num, \ + local_neg_num, only_two_neg_num = run(model, batch) + sentence_ppx_loss += torch.sum(sentence_ppx).data + sentence_ppx_word_loss += torch.sum(sentence_ppx_word).data + sentence_ppx_local_loss += torch.sum(sentence_ppx_local).data + sentence_ppx_only_two_loss += torch.sum(sentence_ppx_only_two).data + + word_cut += word_neg_num + local_cut += local_neg_num + only_two_cut += only_two_neg_num + + if iteration % 50 == 0: + print ("iteration for evaluate:", iteration, "Loss:", decoder_loss.data) + data_len += len(batch['query_text']) + + model.is_inference = False + if model_path != None: + print(' perplexity on test set:', np.exp(sentence_ppx_loss.cpu() / data_len), \ + np.exp(sentence_ppx_word_loss.cpu() / (data_len - int(word_cut))), np.exp(sentence_ppx_local_loss.cpu() / (data_len\ + - int(local_cut))), np.exp(sentence_ppx_only_two_loss.cpu() / (data_len - int(only_two_cut)))) + exit() + print(' perplexity on test set:', np.exp(sentence_ppx_loss.cpu() / data_len), np.exp(sentence_ppx_word_loss.cpu() / \ + (data_len- int(word_cut))), np.exp(sentence_ppx_local_loss.cpu() / (data_len - int(local_cut))), \ + np.exp(sentence_ppx_only_two_loss.cpu() / (data_len - int(only_two_cut)))) + return np.exp(sentence_ppx_loss.cpu() / data_len), np.exp(sentence_ppx_word_loss.cpu() / (data_len- int(word_cut))), \ + np.exp(sentence_ppx_local_loss.cpu() / (data_len - int(local_cut))), np.exp(sentence_ppx_only_two_loss.cpu() / \ + (data_len - int(only_two_cut))) + +def main(): + config = Config('config.yml') + config.list_all_member() + raw_vocab = prepare_data(config) + word2id, entity2id, vocab, embed, entity_vocab, entity_embed, relation_vocab, relation_embed, entity_relation_embed = build_vocab(config.data_dir, raw_vocab, config = config) + data_train, data_test = get_data(config,word2id,entity2id) + + model = use_cuda(ConceptFlow(config, embed, entity_relation_embed)) + + model_optimizer = torch.optim.Adam(model.parameters(), lr = config.lr_rate) + + if not os.path.exists(config.result_dir_name): + os.mkdir(config.result_dir_name) + ppx_f = open(config.result_dir_name + '/result.txt','a') + for name, value in vars(config).items(): + ppx_f.write('%s = %s' % (name, value) + '\n') + + if config.is_train == False: + evaluate(model, data_test, config, word2id, entity2id, 0, model_path = config.test_model_path) + exit() + train(config, model, data_train, data_test, word2id, entity2id, model_optimizer) + +main() diff --git a/training_output/.DS_Store b/training_output/.DS_Store deleted file mode 100644 index 5008ddf..0000000 Binary files a/training_output/.DS_Store and /dev/null differ diff --git a/utils/.DS_Store b/utils/.DS_Store deleted file mode 100644 index 02ed6cf..0000000 Binary files a/utils/.DS_Store and /dev/null differ diff --git a/utils/__pycache__/__init__.cpython-37.pyc b/utils/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index 97df3a6..0000000 Binary files a/utils/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/utils/__pycache__/utils.cpython-37.pyc b/utils/__pycache__/utils.cpython-37.pyc deleted file mode 100644 index 3d82205..0000000 Binary files a/utils/__pycache__/utils.cpython-37.pyc and /dev/null differ