s3nh commited on
Commit
04743be
1 Parent(s): 13ad299

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +404 -0
app.py ADDED
@@ -0,0 +1,404 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import platform
3
+ import random
4
+ import time
5
+ from dataclasses import asdict, dataclass
6
+ from pathlib import Path
7
+
8
+ import gradio as gr
9
+ import psutil
10
+ from about_time import about_time
11
+ from ctransformers import AutoModelForCausalLM
12
+ from dl_hf_model import dl_hf_model
13
+ from loguru import logger
14
+
15
+
16
+ URL = "https://huggingface.co/s3nh/WizardLM-1.0-Uncensored-Llama2-13b-GGML/blob/main/WizardLM-1.0-Uncensored-Llama2-13b.ggmlv3.q4_1.bin" # 4.05G
17
+
18
+ _ = (
19
+ "golay" in platform.node()
20
+ or "okteto" in platform.node()
21
+ or Path("/kaggle").exists()
22
+ # or psutil.cpu_count(logical=False) < 4
23
+ or 1 # run 7b in hf
24
+ )
25
+
26
+ if _:
27
+ url = "https://huggingface.co/s3nh/WizardLM-1.0-Uncensored-Llama2-13b-GGML/blob/main/WizardLM-1.0-Uncensored-Llama2-13b.ggmlv3.q4_1.bin" # 2.87G
28
+
29
+
30
+ prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
31
+ ### Instruction: {user_prompt}
32
+ ### Response:
33
+ """
34
+
35
+ prompt_template = """System: You are a helpful,
36
+ respectful and honest assistant. Always answer as
37
+ helpfully as possible, while being safe. Your answers
38
+ should not include any harmful, unethical, racist,
39
+ sexist, toxic, dangerous, or illegal content. Please
40
+ ensure that your responses are socially unbiased and
41
+ positive in nature. If a question does not make any
42
+ sense, or is not factually coherent, explain why instead
43
+ of answering something not correct. If you don't know
44
+ the answer to a question, please don't share false
45
+ information.
46
+ User: {prompt}
47
+ Assistant: """
48
+
49
+ prompt_template = """System: You are a helpful assistant.
50
+ User: {prompt}
51
+ Assistant: """
52
+
53
+ prompt_template = """Question: {question}
54
+ Answer: Let's work this out in a step by step way to be sure we have the right answer."""
55
+
56
+ prompt_template = """[INST] <>
57
+ You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
58
+ <>
59
+ What NFL team won the Super Bowl in the year Justin Bieber was born?
60
+ [/INST]"""
61
+
62
+ prompt_template = """[INST] <<SYS>>
63
+ You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
64
+ {question} [/INST]
65
+ """
66
+
67
+ prompt_template = """[INST] <<SYS>>
68
+ You are a helpful assistant.
69
+ <</SYS>>
70
+ {question} [/INST]
71
+ """
72
+
73
+ prompt_template = """### HUMAN:
74
+ {question}
75
+ ### RESPONSE:"""
76
+
77
+
78
+ prompt_template = """<|prompt|>:{question}</s>
79
+ <|answer|>:"""
80
+
81
+
82
+ prompt_template = """SYSTEM:
83
+ USER: {question}
84
+ ASSISTANT: """
85
+
86
+
87
+ prompt_template = """
88
+ User: {prompt}
89
+ Assistant: """
90
+
91
+ _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
92
+ stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
93
+
94
+ logger.debug(f"{stop_string=} not used")
95
+
96
+ _ = psutil.cpu_count(logical=False) - 1
97
+ cpu_count: int = int(_) if _ else 1
98
+ logger.debug(f"{cpu_count=}")
99
+
100
+ LLM = None
101
+
102
+ try:
103
+ model_loc, file_size = dl_hf_model(url)
104
+ except Exception as exc_:
105
+ logger.error(exc_)
106
+ raise SystemExit(1) from exc_
107
+
108
+ LLM = AutoModelForCausalLM.from_pretrained(
109
+ model_loc,
110
+ model_type="llama",
111
+ )
112
+
113
+ logger.info(f"done load llm {model_loc=} {file_size=}G")
114
+
115
+ os.environ["TZ"] = "Asia/Shanghai"
116
+ try:
117
+ time.tzset()
118
+
119
+ logger.warning("Windows, cant run time.tzset()")
120
+ except Exception:
121
+ logger.warning("Windows, cant run time.tzset()")
122
+
123
+
124
+ @dataclass
125
+ class GenerationConfig:
126
+ temperature: float = 0.7
127
+ top_k: int = 50
128
+ top_p: float = 0.9
129
+ repetition_penalty: float = 1.0
130
+ max_new_tokens: int = 512
131
+ seed: int = 42
132
+ reset: bool = False
133
+ stream: bool = True
134
+ # threads: int = cpu_count
135
+ # stop: list[str] = field(default_factory=lambda: [stop_string])
136
+
137
+
138
+ def generate(
139
+ question: str,
140
+ llm=LLM,
141
+ config: GenerationConfig = GenerationConfig(),
142
+ ):
143
+ """Run model inference, will return a Generator if streaming is true."""
144
+
145
+
146
+ prompt = prompt_template.format(question=question)
147
+
148
+ return llm(
149
+ prompt,
150
+ **asdict(config),
151
+ )
152
+
153
+
154
+ logger.debug(f"{asdict(GenerationConfig())=}")
155
+
156
+
157
+ def user(user_message, history):
158
+ history.append([user_message, None])
159
+ return user_message, history
160
+
161
+
162
+ def user1(user_message, history):
163
+ history.append([user_message, None])
164
+ return "", history
165
+
166
+ def bot_(history):
167
+ user_message = history[-1][0]
168
+ resp = random.choice(["How are you?", "I love you", "I'm very hungry"])
169
+ bot_message = user_message + ": " + resp
170
+ history[-1][1] = ""
171
+ for character in bot_message:
172
+ history[-1][1] += character
173
+ time.sleep(0.02)
174
+ yield history
175
+
176
+ history[-1][1] = resp
177
+ yield history
178
+
179
+
180
+ def bot(history):
181
+ user_message = history[-1][0]
182
+ response = []
183
+
184
+ logger.debug(f"{user_message=}")
185
+
186
+ with about_time() as atime:
187
+ flag = 1
188
+ prefix = ""
189
+ then = time.time()
190
+
191
+ logger.debug("about to generate")
192
+
193
+ config = GenerationConfig(reset=True)
194
+ for elm in generate(user_message, config=config):
195
+ if flag == 1:
196
+ logger.debug("in the loop")
197
+ prefix = f"({time.time() - then:.2f}s) "
198
+ flag = 0
199
+ print(prefix, end="", flush=True)
200
+ logger.debug(f"{prefix=}")
201
+ print(elm, end="", flush=True)
202
+
203
+ response.append(elm)
204
+ history[-1][1] = prefix + "".join(response)
205
+ yield history
206
+
207
+ _ = (
208
+ f"(time elapsed: {atime.duration_human}, "
209
+ f"{atime.duration/len(''.join(response)):.2f}s/char)"
210
+ )
211
+
212
+ history[-1][1] = "".join(response) + f"\n{_}"
213
+ yield history
214
+
215
+
216
+ def predict_api(prompt):
217
+ logger.debug(f"{prompt=}")
218
+ try:
219
+ # user_prompt = prompt
220
+ config = GenerationConfig(
221
+ temperature=0.2,
222
+ top_k=10,
223
+ top_p=0.9,
224
+ repetition_penalty=1.0,
225
+ max_new_tokens=512, # adjust as needed
226
+ seed=42,
227
+ reset=True,
228
+ stream=False,
229
+ )
230
+
231
+ response = generate(
232
+ prompt,
233
+ config=config,
234
+ )
235
+
236
+ logger.debug(f"api: {response=}")
237
+ except Exception as exc:
238
+ logger.error(exc)
239
+ response = f"{exc=}"
240
+ return response
241
+
242
+
243
+ css = """
244
+ .importantButton {
245
+ background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
246
+ border: none !important;
247
+ }
248
+ .importantButton:hover {
249
+ background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
250
+ border: none !important;
251
+ }
252
+ .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
253
+ .xsmall {font-size: x-small;}
254
+ """
255
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
256
+ examples_list = [
257
+ ["Send an email requesting that people use language models responsibly."],
258
+ ["Write a shouting match between Julius Caesar and Napoleon"],
259
+ ["Write a theory to explain why cat never existed"],
260
+ ["write a story about a grain of sand as it watches millions of years go by"],
261
+ ["What are 3 popular chess openings?"],
262
+ ["write a conversation between the sun and pluto"],
263
+ ["Did you know that Yann LeCun dropped a rap album last year? We listened to it andhere’s what we thought:"],
264
+ ]
265
+
266
+ logger.info("start block")
267
+
268
+ with gr.Blocks(
269
+ title=f"{Path(model_loc).name}",
270
+ theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
271
+ css=css,
272
+ ) as block:
273
+ # buff_var = gr.State("")
274
+ with gr.Accordion("🎈 Info", open=False):
275
+ # gr.HTML(
276
+ # """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
277
+ # )
278
+ gr.Markdown(
279
+ f"""<h5><center>{Path(model_loc).name}</center></h4>
280
+ Most examples are meant for another model.
281
+ You probably should try to test
282
+ some related prompts.""",
283
+ elem_classes="xsmall",
284
+ )
285
+
286
+ # chatbot = gr.Chatbot().style(height=700) # 500
287
+ chatbot = gr.Chatbot(height=500)
288
+
289
+ # buff = gr.Textbox(show_label=False, visible=True)
290
+
291
+ with gr.Row():
292
+ with gr.Column(scale=5):
293
+ msg = gr.Textbox(
294
+ label="Chat Message Box",
295
+ placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
296
+ show_label=False,
297
+ # container=False,
298
+ lines=6,
299
+ max_lines=30,
300
+ show_copy_button=True,
301
+ # ).style(container=False)
302
+ )
303
+ with gr.Column(scale=1, min_width=50):
304
+ with gr.Row():
305
+ submit = gr.Button("Submit", elem_classes="xsmall")
306
+ stop = gr.Button("Stop", visible=True)
307
+ clear = gr.Button("Clear History", visible=True)
308
+ with gr.Row(visible=False):
309
+ with gr.Accordion("Advanced Options:", open=False):
310
+ with gr.Row():
311
+ with gr.Column(scale=2):
312
+ system = gr.Textbox(
313
+ label="System Prompt",
314
+ value=prompt_template,
315
+ show_label=False,
316
+ container=False,
317
+ # ).style(container=False)
318
+ )
319
+ with gr.Column():
320
+ with gr.Row():
321
+ change = gr.Button("Change System Prompt")
322
+ reset = gr.Button("Reset System Prompt")
323
+
324
+ with gr.Accordion("Example Inputs", open=True):
325
+ examples = gr.Examples(
326
+ examples=examples_list,
327
+ inputs=[msg],
328
+ examples_per_page=40,
329
+ )
330
+
331
+ # with gr.Row():
332
+ with gr.Accordion("Disclaimer", open=True):
333
+ _ = Path(model_loc).name
334
+ gr.Markdown(
335
+ "Disclaimer: I AM NOT RESPONSIBLE FOR ANY PROMPT PROVIDED BY USER AND PROMPT RETURNED FROM THE MODEL. THIS APP SHOULD BE USED FOR EDUCATIONAL PURPOSE"
336
+ "WITHOUT ANY OFFENSIVE, AGGRESIVE INTENTS. {_} can produce factually incorrect output, and should not be relied on to produce "
337
+ f"factually accurate information. {_} was trained on various public datasets; while great efforts "
338
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
339
+ "biased, or otherwise offensive outputs.",
340
+ elem_classes=["disclaimer"],
341
+ )
342
+
343
+ msg_submit_event = msg.submit(
344
+ # fn=conversation.user_turn,
345
+ fn=user,
346
+ inputs=[msg, chatbot],
347
+ outputs=[msg, chatbot],
348
+ queue=True,
349
+ show_progress="full",
350
+ # api_name=None,
351
+ ).then(bot, chatbot, chatbot, queue=True)
352
+ submit_click_event = submit.click(
353
+ # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
354
+ fn=user1, # clear msg
355
+ inputs=[msg, chatbot],
356
+ outputs=[msg, chatbot],
357
+ queue=True,
358
+ # queue=False,
359
+ show_progress="full",
360
+ # api_name=None,
361
+ ).then(bot, chatbot, chatbot, queue=True)
362
+ stop.click(
363
+ fn=None,
364
+ inputs=None,
365
+ outputs=None,
366
+ cancels=[msg_submit_event, submit_click_event],
367
+ queue=False,
368
+ )
369
+ clear.click(lambda: None, None, chatbot, queue=False)
370
+
371
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
372
+ input_text = gr.Text()
373
+ api_btn = gr.Button("Go", variant="primary")
374
+ out_text = gr.Text()
375
+
376
+ api_btn.click(
377
+ predict_api,
378
+ input_text,
379
+ out_text,
380
+ api_name="api",
381
+ )
382
+
383
+ # block.load(update_buff, [], buff, every=1)
384
+ # block.load(update_buff, [buff_var], [buff_var, buff], every=1)
385
+
386
+ # concurrency_count=5, max_size=20
387
+ # max_size=36, concurrency_count=14
388
+ # CPU cpu_count=2 16G, model 7G
389
+ # CPU UPGRADE cpu_count=8 32G, model 7G
390
+
391
+ # does not work
392
+ _ = """
393
+ # _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
394
+ # concurrency_count = max(_, 1)
395
+ if psutil.cpu_count(logical=False) >= 8:
396
+ # concurrency_count = max(int(32 / file_size) - 1, 1)
397
+ else:
398
+ # concurrency_count = max(int(16 / file_size) - 1, 1)
399
+ # """
400
+
401
+ concurrency_count = 1
402
+ logger.info(f"{concurrency_count=}")
403
+
404
+ block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)