songxxzp commited on
Commit
7b7d06c
1 Parent(s): 65546ae

Use parallel kernel

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -8,7 +8,7 @@ mem = psutil.virtual_memory()
8
 
9
  tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
10
 
11
- model = AutoModelForSeq2SeqLM.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True).cpu().float()
12
 
13
  def chat(query, history=[]):
14
  _, history = model.chat(tokenizer, query, history, max_length=512)
 
8
 
9
  tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
10
 
11
+ model = AutoModelForSeq2SeqLM.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True).quantize(bits=4, compile_parallel_kernel=True, parallel_num=2).cpu().float()
12
 
13
  def chat(query, history=[]):
14
  _, history = model.chat(tokenizer, query, history, max_length=512)