1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
| import logging
import os
import sys
import fire
import numpy as np
from datasets import load_dataset
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import (
T5ForConditionalGeneration,
AutoTokenizer,
)
sys.path.append('../')
from utils import EvaluationDataset, evaluation_results, get_metrics_results
def main(log_dir: str, checkpoint_path: str, data_path: str, item_indexing: str, task: str,
dataset: str, cutoff: int, test_prompt: str, eval_batch_size: int, metrics: str):
log_file = os.path.join(log_dir, dataset,
checkpoint_path.replace('.', '').replace('/', '_') + '.log')
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=log_file, level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
model = T5ForConditionalGeneration.from_pretrained(checkpoint_path)
tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
tokenizer.pad_token_id = (
0
)
tokenizer.padding_side = "left"
test_data_file = os.path.join(data_path, dataset,
f'{dataset}_{task}_{item_indexing}_test_{test_prompt}.json')
logging.info("test_data_file=" + test_data_file)
test_data = load_dataset("json", data_files=test_data_file, field='data')
model.eval()
metrics = list(metrics)
generate_num = max([int(m.split('@')[1]) for m in metrics])
task_list = np.unique(test_data['train']['task'])
for t in task_list:
logging.info(f'testing on {t}')
subset_data = test_data.filter(lambda example: example['task'] == t)
dataset = EvaluationDataset(subset_data['train'], tokenizer, cutoff)
dataloader = DataLoader(dataset, batch_size=eval_batch_size, shuffle=False)
test_total = 0
metrics_res = np.array([0.0] * len(metrics))
for batch in tqdm(dataloader):
"""
下面是一个batch的案例：
{'input_ids': tensor([[ 3, 21419, 12587, ..., 0, 0, 0],
...,
[ 3, 21419, 12587, ..., 0, 0, 0]]),
'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0],
...,
[1, 1, 1, ..., 0, 0, 0]]),
'label': tensor([[12587, 2118, 834, 22504, 2577, 1, 0],
[12587, 2118, 834, 19993, 4867, 1, 0],
...,
[12587, 2118, 834, 19993, 5062, 1, 0]])}
"""
prediction = model.generate(
input_ids=batch["input_ids"],
attention_mask=batch["attention_mask"],
max_length=30,
num_beams=generate_num,
num_return_sequences=generate_num,
output_scores=True,
return_dict_in_generate=True,
)
output_ids = batch['label']
prediction_ids = prediction["sequences"]
prediction_scores = prediction["sequences_scores"]
gold_sents = tokenizer.batch_decode(
output_ids, skip_special_tokens=True
)
generated_sents = tokenizer.batch_decode(
prediction_ids, skip_special_tokens=True
)
rel_results = evaluation_results(generated_sents, gold_sents, prediction_scores, generate_num)
test_total += len(rel_results)
metrics_res += get_metrics_results(rel_results, metrics)
metrics_res /= test_total
for i in range(len(metrics)):
logging.info(f'{metrics[i]}: {metrics_res[i]}')
if __name__ == "__main__":
fire.Fire(main)
|
微信支付
支付宝
