-
Notifications
You must be signed in to change notification settings - Fork 11
/
Copy pathllama_infer.py
73 lines (62 loc) · 2.99 KB
/
llama_infer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import argparse
from utils import load_hyperparam, convert_normal_parameter_to_int8, load_model
from model.tokenize import Tokenizer
from model.llama import *
from generate import LmGeneration
if __name__ == '__main__':
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--load_model_path", default=None, type=str,
help="Path of the input model.")
parser.add_argument("--test_path", type=str, required=True,
help="Path of the testset.")
parser.add_argument("--prediction_path", type=str, required=True,
help="Path of the prediction file.")
parser.add_argument("--config_path", type=str, required=True,
help="Path of the config file.")
parser.add_argument("--batch_size", type=int, default=1,
help="Batch size.")
parser.add_argument("--world_size", type=int, default=1,
help="the number of gpus.")
parser.add_argument("--seq_length", type=int, default=128,
help="Sequence length.")
parser.add_argument("--use_int8", action="store_true")
parser.add_argument("--top_k", type=int, default=10)
parser.add_argument("--top_p", type=float, default=1)
parser.add_argument("--temperature", type=float, default=0.85)
parser.add_argument("--repetition_penalty_range", type=int, default=1024)
parser.add_argument("--repetition_penalty_slope", type=float, default=0)
parser.add_argument("--repetition_penalty", type=float, default=1.15)
parser.add_argument("--spm_model_path", default=None, type=str,
help="Path of the sentence piece model.")
args = parser.parse_args()
args = load_hyperparam(args)
args.tokenizer = Tokenizer(model_path=args.spm_model_path)
args.vocab_size = args.tokenizer.sp_model.vocab_size()
torch.set_default_tensor_type(torch.HalfTensor)
model = LLaMa(args)
torch.set_default_tensor_type(torch.FloatTensor)
model = load_model(model, args.load_model_path)
model.eval()
# use multi-gpu tensor parallel
if args.world_size > 1:
import tensor_parallel as tp
gpus = ["cuda:" + str(i) for i in range(args.world_size)]
if args.use_int8:
model = tp.tensor_parallel(model, gpus, delay_init=True)
model = convert_normal_parameter_to_int8(model)
else:
model = tp.tensor_parallel(model, gpus)
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
lm_generation = LmGeneration(model, args.tokenizer)
prompts = []
with open(args.test_path, 'r', encoding='utf-8') as f:
for line in f:
prompts.append(line)
with torch.no_grad():
result = lm_generation.generate(args, prompts)
with open(args.prediction_path, 'w', encoding='utf-8') as f:
for res in result:
f.write(res + '\n')
f.write('\n')