Update app.py
Browse files
app.py
CHANGED
@@ -4,9 +4,9 @@ import os
|
|
4 |
import pandas as pd
|
5 |
from typing import List, Tuple
|
6 |
|
7 |
-
# LLM
|
8 |
LLM_MODELS = {
|
9 |
-
"Cohere c4ai-crp-08-2024": "CohereForAI/c4ai-command-r-plus-08-2024", #
|
10 |
"Meta Llama3.3-70B": "meta-llama/Llama-3.3-70B-Instruct",
|
11 |
"Mistral Nemo 2407": "mistralai/Mistral-Nemo-Instruct-2407",
|
12 |
"Alibaba Qwen QwQ-32B": "Qwen/QwQ-32B-Preview"
|
@@ -16,19 +16,17 @@ def get_client(model_name):
|
|
16 |
return InferenceClient(LLM_MODELS[model_name], token=os.getenv("HF_TOKEN"))
|
17 |
|
18 |
def analyze_file_content(content, file_type):
|
19 |
-
"""
|
20 |
if file_type in ['parquet', 'csv']:
|
21 |
try:
|
22 |
-
# λ°μ΄ν°μ
ꡬ쑰 λΆμ
|
23 |
lines = content.split('\n')
|
24 |
header = lines[0]
|
25 |
columns = header.count('|') - 1
|
26 |
-
rows = len(lines) - 3
|
27 |
-
return f"
|
28 |
except:
|
29 |
-
return "
|
30 |
|
31 |
-
# ν
μ€νΈ/μ½λ νμΌμ κ²½μ°
|
32 |
lines = content.split('\n')
|
33 |
total_lines = len(lines)
|
34 |
non_empty_lines = len([line for line in lines if line.strip()])
|
@@ -37,11 +35,11 @@ def analyze_file_content(content, file_type):
|
|
37 |
functions = len([line for line in lines if 'def ' in line])
|
38 |
classes = len([line for line in lines if 'class ' in line])
|
39 |
imports = len([line for line in lines if 'import ' in line or 'from ' in line])
|
40 |
-
return f"
|
41 |
|
42 |
paragraphs = content.count('\n\n') + 1
|
43 |
words = len(content.split())
|
44 |
-
return f"
|
45 |
|
46 |
def read_uploaded_file(file):
|
47 |
if file is None:
|
@@ -54,32 +52,28 @@ def read_uploaded_file(file):
|
|
54 |
content = df.head(10).to_markdown(index=False)
|
55 |
return content, "parquet"
|
56 |
elif file_ext == '.csv':
|
57 |
-
# CSV νμΌ μ½κΈ° μ λ€μν μΈμ½λ© μλ
|
58 |
encodings = ['utf-8', 'cp949', 'euc-kr', 'latin1']
|
59 |
for encoding in encodings:
|
60 |
try:
|
61 |
df = pd.read_csv(file.name, encoding=encoding)
|
62 |
-
content = f"
|
63 |
-
content += f"\n
|
64 |
-
content += f"-
|
65 |
-
content += f"-
|
66 |
-
content += f"-
|
67 |
-
|
68 |
-
content += f"\n컬λΌλ³ λ°μ΄ν° νμ
:\n"
|
69 |
for col, dtype in df.dtypes.items():
|
70 |
content += f"- {col}: {dtype}\n"
|
71 |
-
# κ²°μΈ‘μΉ μ 보 μΆκ°
|
72 |
null_counts = df.isnull().sum()
|
73 |
if null_counts.any():
|
74 |
-
content += f"\n
|
75 |
for col, null_count in null_counts[null_counts > 0].items():
|
76 |
-
content += f"- {col}: {null_count}
|
77 |
return content, "csv"
|
78 |
except UnicodeDecodeError:
|
79 |
continue
|
80 |
-
raise UnicodeDecodeError(f"
|
81 |
else:
|
82 |
-
# ν
μ€νΈ νμΌ μ½κΈ° μλ
|
83 |
encodings = ['utf-8', 'cp949', 'euc-kr', 'latin1']
|
84 |
for encoding in encodings:
|
85 |
try:
|
@@ -88,9 +82,9 @@ def read_uploaded_file(file):
|
|
88 |
return content, "text"
|
89 |
except UnicodeDecodeError:
|
90 |
continue
|
91 |
-
raise UnicodeDecodeError(f"
|
92 |
except Exception as e:
|
93 |
-
return f"
|
94 |
|
95 |
def format_history(history):
|
96 |
formatted_history = []
|
@@ -101,17 +95,16 @@ def format_history(history):
|
|
101 |
return formatted_history
|
102 |
|
103 |
def chat(message, history, uploaded_file, model_name, system_message="", max_tokens=4000, temperature=0.7, top_p=0.9):
|
104 |
-
system_prefix = """
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
105 |
|
106 |
-
|
107 |
-
2. μ£Όμ λ΄μ©κ³Ό ν¨ν΄ λΆμ
|
108 |
-
3. λ°μ΄ν°μ νΉμ§κ³Ό μλ―Έ
|
109 |
-
- λ°μ΄ν°μ
μ κ²½μ°: 컬λΌμ μλ―Έ, λ°μ΄ν° νμ
, κ°μ λΆν¬
|
110 |
-
- ν
μ€νΈ/μ½λμ κ²½μ°: ꡬ쑰μ νΉμ§, μ£Όμ ν¨ν΄
|
111 |
-
4. μ μ¬μ νμ© λ°©μ
|
112 |
-
5. λ°μ΄ν° νμ§ λ° κ°μ κ°λ₯ν λΆλΆ
|
113 |
-
|
114 |
-
μ λ¬Έκ°μ κ΄μ μμ μμΈνκ³ κ΅¬μ‘°μ μΈ λΆμμ μ 곡νλ, μ΄ν΄νκΈ° μ½κ² μ€λͺ
νμΈμ. λΆμ κ²°κ³Όλ Markdown νμμΌλ‘ μμ±νκ³ , κ°λ₯ν ν ꡬ체μ μΈ μμλ₯Ό ν¬ν¨νμΈμ."""
|
115 |
|
116 |
if uploaded_file:
|
117 |
content, file_type = read_uploaded_file(uploaded_file)
|
@@ -119,24 +112,23 @@ def chat(message, history, uploaded_file, model_name, system_message="", max_tok
|
|
119 |
yield "", history + [[message, content]]
|
120 |
return
|
121 |
|
122 |
-
# νμΌ λ΄μ© λΆμ λ° κ΅¬μ‘°μ μμ½
|
123 |
file_summary = analyze_file_content(content, file_type)
|
124 |
|
125 |
if file_type in ['parquet', 'csv']:
|
126 |
-
system_message += f"\n\
|
127 |
else:
|
128 |
-
system_message += f"\n\
|
129 |
|
130 |
-
if message == "
|
131 |
-
message = f"""[
|
132 |
|
133 |
-
|
134 |
-
1.
|
135 |
-
2.
|
136 |
-
3.
|
137 |
-
4.
|
138 |
-
5.
|
139 |
-
6.
|
140 |
|
141 |
messages = [{"role": "system", "content": f"{system_prefix} {system_message}"}]
|
142 |
messages.extend(format_history(history))
|
@@ -159,46 +151,56 @@ def chat(message, history, uploaded_file, model_name, system_message="", max_tok
|
|
159 |
yield "", history + [[message, partial_message]]
|
160 |
|
161 |
except Exception as e:
|
162 |
-
error_msg = f"
|
163 |
yield "", history + [[message, error_msg]]
|
164 |
|
165 |
css = """
|
166 |
footer {visibility: hidden}
|
167 |
"""
|
168 |
|
169 |
-
with gr.Blocks(theme="Yntec/HaleyCH_Theme_Orange", css=css) as demo:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
170 |
with gr.Row():
|
171 |
with gr.Column(scale=2):
|
172 |
-
chatbot = gr.Chatbot(height=600)
|
173 |
msg = gr.Textbox(
|
174 |
-
label="
|
175 |
show_label=False,
|
176 |
-
placeholder="
|
177 |
container=False
|
178 |
)
|
179 |
-
clear = gr.ClearButton([msg, chatbot])
|
180 |
|
181 |
with gr.Column(scale=1):
|
182 |
model_name = gr.Radio(
|
183 |
choices=list(LLM_MODELS.keys()),
|
184 |
-
value="Cohere c4ai-crp-08-2024",
|
185 |
-
label="
|
186 |
-
info="
|
187 |
)
|
188 |
|
189 |
file_upload = gr.File(
|
190 |
-
label="
|
|
|
191 |
file_types=["text", ".csv", ".parquet"],
|
192 |
type="filepath"
|
193 |
)
|
194 |
|
195 |
-
with gr.Accordion("
|
196 |
-
system_message = gr.Textbox(label="System Message", value="")
|
197 |
-
max_tokens = gr.Slider(minimum=1, maximum=8000, value=4000, label="Max Tokens")
|
198 |
-
temperature = gr.Slider(minimum=0, maximum=1, value=0.7, label="Temperature")
|
199 |
-
top_p = gr.Slider(minimum=0, maximum=1, value=0.9, label="Top P")
|
200 |
|
201 |
-
#
|
202 |
msg.submit(
|
203 |
chat,
|
204 |
inputs=[msg, chatbot, file_upload, model_name, system_message, max_tokens, temperature, top_p],
|
@@ -210,26 +212,26 @@ with gr.Blocks(theme="Yntec/HaleyCH_Theme_Orange", css=css) as demo:
|
|
210 |
[msg]
|
211 |
)
|
212 |
|
213 |
-
#
|
214 |
file_upload.change(
|
215 |
chat,
|
216 |
-
inputs=[gr.Textbox(value="
|
217 |
outputs=[msg, chatbot],
|
218 |
queue=True
|
219 |
)
|
220 |
|
221 |
-
#
|
222 |
gr.Examples(
|
223 |
examples=[
|
224 |
-
["
|
225 |
-
["
|
226 |
-
["
|
227 |
-
["
|
228 |
-
["
|
229 |
-
["
|
230 |
],
|
231 |
inputs=msg,
|
232 |
)
|
233 |
|
234 |
if __name__ == "__main__":
|
235 |
-
demo.launch()
|
|
|
4 |
import pandas as pd
|
5 |
from typing import List, Tuple
|
6 |
|
7 |
+
# LLM Models Definition
|
8 |
LLM_MODELS = {
|
9 |
+
"Cohere c4ai-crp-08-2024": "CohereForAI/c4ai-command-r-plus-08-2024", # Default
|
10 |
"Meta Llama3.3-70B": "meta-llama/Llama-3.3-70B-Instruct",
|
11 |
"Mistral Nemo 2407": "mistralai/Mistral-Nemo-Instruct-2407",
|
12 |
"Alibaba Qwen QwQ-32B": "Qwen/QwQ-32B-Preview"
|
|
|
16 |
return InferenceClient(LLM_MODELS[model_name], token=os.getenv("HF_TOKEN"))
|
17 |
|
18 |
def analyze_file_content(content, file_type):
|
19 |
+
"""Analyze file content and return structural summary"""
|
20 |
if file_type in ['parquet', 'csv']:
|
21 |
try:
|
|
|
22 |
lines = content.split('\n')
|
23 |
header = lines[0]
|
24 |
columns = header.count('|') - 1
|
25 |
+
rows = len(lines) - 3
|
26 |
+
return f"π Dataset Structure: {columns} columns, {rows} data samples"
|
27 |
except:
|
28 |
+
return "β Dataset structure analysis failed"
|
29 |
|
|
|
30 |
lines = content.split('\n')
|
31 |
total_lines = len(lines)
|
32 |
non_empty_lines = len([line for line in lines if line.strip()])
|
|
|
35 |
functions = len([line for line in lines if 'def ' in line])
|
36 |
classes = len([line for line in lines if 'class ' in line])
|
37 |
imports = len([line for line in lines if 'import ' in line or 'from ' in line])
|
38 |
+
return f"π» Code Structure: {total_lines} lines (Functions: {functions}, Classes: {classes}, Imports: {imports})"
|
39 |
|
40 |
paragraphs = content.count('\n\n') + 1
|
41 |
words = len(content.split())
|
42 |
+
return f"π Document Structure: {total_lines} lines, {paragraphs} paragraphs, ~{words} words"
|
43 |
|
44 |
def read_uploaded_file(file):
|
45 |
if file is None:
|
|
|
52 |
content = df.head(10).to_markdown(index=False)
|
53 |
return content, "parquet"
|
54 |
elif file_ext == '.csv':
|
|
|
55 |
encodings = ['utf-8', 'cp949', 'euc-kr', 'latin1']
|
56 |
for encoding in encodings:
|
57 |
try:
|
58 |
df = pd.read_csv(file.name, encoding=encoding)
|
59 |
+
content = f"π Data Preview:\n{df.head(10).to_markdown(index=False)}\n\n"
|
60 |
+
content += f"\nπ Data Information:\n"
|
61 |
+
content += f"- Total Rows: {len(df)}\n"
|
62 |
+
content += f"- Total Columns: {len(df.columns)}\n"
|
63 |
+
content += f"- Column List: {', '.join(df.columns)}\n"
|
64 |
+
content += f"\nπ Column Data Types:\n"
|
|
|
65 |
for col, dtype in df.dtypes.items():
|
66 |
content += f"- {col}: {dtype}\n"
|
|
|
67 |
null_counts = df.isnull().sum()
|
68 |
if null_counts.any():
|
69 |
+
content += f"\nβ οΈ Missing Values:\n"
|
70 |
for col, null_count in null_counts[null_counts > 0].items():
|
71 |
+
content += f"- {col}: {null_count} missing\n"
|
72 |
return content, "csv"
|
73 |
except UnicodeDecodeError:
|
74 |
continue
|
75 |
+
raise UnicodeDecodeError(f"β Unable to read file with supported encodings ({', '.join(encodings)})")
|
76 |
else:
|
|
|
77 |
encodings = ['utf-8', 'cp949', 'euc-kr', 'latin1']
|
78 |
for encoding in encodings:
|
79 |
try:
|
|
|
82 |
return content, "text"
|
83 |
except UnicodeDecodeError:
|
84 |
continue
|
85 |
+
raise UnicodeDecodeError(f"β Unable to read file with supported encodings ({', '.join(encodings)})")
|
86 |
except Exception as e:
|
87 |
+
return f"β Error reading file: {str(e)}", "error"
|
88 |
|
89 |
def format_history(history):
|
90 |
formatted_history = []
|
|
|
95 |
return formatted_history
|
96 |
|
97 |
def chat(message, history, uploaded_file, model_name, system_message="", max_tokens=4000, temperature=0.7, top_p=0.9):
|
98 |
+
system_prefix = """You are a file analysis expert. Analyze the uploaded file in depth from the following perspectives:
|
99 |
+
1. π Overall structure and composition
|
100 |
+
2. π Key content and pattern analysis
|
101 |
+
3. π Data characteristics and meaning
|
102 |
+
- For datasets: Column meanings, data types, value distributions
|
103 |
+
- For text/code: Structural features, main patterns
|
104 |
+
4. π‘ Potential applications
|
105 |
+
5. β¨ Data quality and areas for improvement
|
106 |
|
107 |
+
Provide detailed and structured analysis from an expert perspective, but explain in an easy-to-understand way. Format the analysis results in Markdown and include specific examples where possible."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
108 |
|
109 |
if uploaded_file:
|
110 |
content, file_type = read_uploaded_file(uploaded_file)
|
|
|
112 |
yield "", history + [[message, content]]
|
113 |
return
|
114 |
|
|
|
115 |
file_summary = analyze_file_content(content, file_type)
|
116 |
|
117 |
if file_type in ['parquet', 'csv']:
|
118 |
+
system_message += f"\n\nFile Content:\n```markdown\n{content}\n```"
|
119 |
else:
|
120 |
+
system_message += f"\n\nFile Content:\n```\n{content}\n```"
|
121 |
|
122 |
+
if message == "Starting file analysis...":
|
123 |
+
message = f"""[Structure Analysis] {file_summary}
|
124 |
|
125 |
+
Please provide detailed analysis from these perspectives:
|
126 |
+
1. π Overall file structure and format
|
127 |
+
2. π Key content and component analysis
|
128 |
+
3. π Data/content characteristics and patterns
|
129 |
+
4. β Quality and completeness evaluation
|
130 |
+
5. π‘ Suggested improvements
|
131 |
+
6. π― Practical applications and recommendations"""
|
132 |
|
133 |
messages = [{"role": "system", "content": f"{system_prefix} {system_message}"}]
|
134 |
messages.extend(format_history(history))
|
|
|
151 |
yield "", history + [[message, partial_message]]
|
152 |
|
153 |
except Exception as e:
|
154 |
+
error_msg = f"β Inference error: {str(e)}"
|
155 |
yield "", history + [[message, error_msg]]
|
156 |
|
157 |
css = """
|
158 |
footer {visibility: hidden}
|
159 |
"""
|
160 |
|
161 |
+
with gr.Blocks(theme="Yntec/HaleyCH_Theme_Orange", css=css, title="EveryChat π€") as demo:
|
162 |
+
gr.HTML(
|
163 |
+
"""
|
164 |
+
<div style="text-align: center; max-width: 800px; margin: 0 auto;">
|
165 |
+
<h1 style="font-size: 3em; font-weight: 600; margin: 0.5em;">EveryChat π€</h1>
|
166 |
+
<h3 style="font-size: 1.2em; margin: 1em;">Your Intelligent File Analysis Assistant π</h3>
|
167 |
+
</div>
|
168 |
+
"""
|
169 |
+
)
|
170 |
+
|
171 |
with gr.Row():
|
172 |
with gr.Column(scale=2):
|
173 |
+
chatbot = gr.Chatbot(height=600, label="Chat Interface π¬")
|
174 |
msg = gr.Textbox(
|
175 |
+
label="Type your message",
|
176 |
show_label=False,
|
177 |
+
placeholder="Ask me anything about the uploaded file... π",
|
178 |
container=False
|
179 |
)
|
180 |
+
clear = gr.ClearButton([msg, chatbot], label="Clear Chat ποΈ")
|
181 |
|
182 |
with gr.Column(scale=1):
|
183 |
model_name = gr.Radio(
|
184 |
choices=list(LLM_MODELS.keys()),
|
185 |
+
value="Cohere c4ai-crp-08-2024",
|
186 |
+
label="Select LLM Model π€",
|
187 |
+
info="Choose your preferred AI model"
|
188 |
)
|
189 |
|
190 |
file_upload = gr.File(
|
191 |
+
label="Upload File π",
|
192 |
+
info="Support: Text, Code, CSV, Parquet files",
|
193 |
file_types=["text", ".csv", ".parquet"],
|
194 |
type="filepath"
|
195 |
)
|
196 |
|
197 |
+
with gr.Accordion("Advanced Settings βοΈ", open=False):
|
198 |
+
system_message = gr.Textbox(label="System Message π", value="")
|
199 |
+
max_tokens = gr.Slider(minimum=1, maximum=8000, value=4000, label="Max Tokens π")
|
200 |
+
temperature = gr.Slider(minimum=0, maximum=1, value=0.7, label="Temperature π‘οΈ")
|
201 |
+
top_p = gr.Slider(minimum=0, maximum=1, value=0.9, label="Top P π")
|
202 |
|
203 |
+
# Event bindings
|
204 |
msg.submit(
|
205 |
chat,
|
206 |
inputs=[msg, chatbot, file_upload, model_name, system_message, max_tokens, temperature, top_p],
|
|
|
212 |
[msg]
|
213 |
)
|
214 |
|
215 |
+
# Auto-analysis on file upload
|
216 |
file_upload.change(
|
217 |
chat,
|
218 |
+
inputs=[gr.Textbox(value="Starting file analysis..."), chatbot, file_upload, model_name, system_message, max_tokens, temperature, top_p],
|
219 |
outputs=[msg, chatbot],
|
220 |
queue=True
|
221 |
)
|
222 |
|
223 |
+
# Example queries
|
224 |
gr.Examples(
|
225 |
examples=[
|
226 |
+
["Please explain the overall structure and features of the file in detail π"],
|
227 |
+
["Analyze the main patterns and characteristics of this file π"],
|
228 |
+
["Evaluate the file's quality and potential improvements π‘"],
|
229 |
+
["How can we practically utilize this file? π―"],
|
230 |
+
["Summarize the main content and derive key insights β¨"],
|
231 |
+
["Please continue with more detailed analysis π"],
|
232 |
],
|
233 |
inputs=msg,
|
234 |
)
|
235 |
|
236 |
if __name__ == "__main__":
|
237 |
+
demo.launch()
|