Update app.py

app.py

@@ -7,8 +7,17 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import lru_cache
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
+from openai import OpenAI
+from bs4 import BeautifulSoup
 
+ACCESS_TOKEN = os.getenv("HF_TOKEN")
+if not ACCESS_TOKEN:
+    raise ValueError("HF_TOKEN environment variable is not set")
 
+client = OpenAI(
+    base_url="https://api-inference.huggingface.co/v1/",
+    api_key=ACCESS_TOKEN,
+)
 
 MAX_COUNTRY_RESULTS = 100    # 국가별 최대 결과 수
 MAX_GLOBAL_RESULTS = 1000    # 전세계 최대 결과 수
@@ -825,6 +834,88 @@ footer {visibility: hidden;}
 }
 """
 
+
+def get_article_content(url):
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        response = requests.get(url, headers=headers)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # 일반적인 기사 본문 컨테이너 검색
+        article_body = None
+        possible_content_elements = [
+            soup.find('article'),
+            soup.find('div', class_='article-body'),
+            soup.find('div', class_='content'),
+            soup.find('div', {'id': 'article-body'})
+        ]
+        
+        for element in possible_content_elements:
+            if element:
+                article_body = element
+                break
+        
+        if article_body:
+            # 불필요한 요소 제거
+            for tag in article_body.find_all(['script', 'style', 'nav', 'header', 'footer']):
+                tag.decompose()
+            
+            content = ' '.join([p.get_text().strip() for p in article_body.find_all('p') if p.get_text().strip()])
+        else:
+            content = ' '.join([p.get_text().strip() for p in soup.find_all('p') if p.get_text().strip()])
+        
+        return content
+    except Exception as e:
+        return f"Error crawling content: {str(e)}"
+
+def respond(
+    url,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    if not url.startswith('http'):
+        return "Please enter a valid URL"
+    
+    # 기사 내용 추출
+    article_content = get_article_content(url)
+    
+    # 번역 요청을 위한 프롬프트 구성
+    translation_prompt = f"""Please translate the following article to Korean. 
+    Maintain the original meaning and context while making it natural in Korean.
+    
+    Article: {article_content}
+    
+    Korean translation:"""
+    
+    messages = [
+        {"role": "system", "content": system_message or "You are a helpful translator that translates articles to Korean."},
+        {"role": "user", "content": translation_prompt}
+    ]
+
+    response = ""
+    
+    try:
+        for message in client.chat.completions.create(
+            model="CohereForAI/c4ai-command-r-plus-08-2024",
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
+            messages=messages,
+        ):
+            token = message.choices[0].delta.content
+            if token:
+                response += token
+                yield response
+    except Exception as e:
+        yield f"Translation error: {str(e)}"
+
+
 with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as iface:
     with gr.Tabs():
         # 국가별 탭
@@ -902,6 +993,66 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
                                 'index': i,
                             })
 
+    
+    # AI 번역 탭 추가
+        with gr.Tab("AI 번역"):
+            gr.Markdown("뉴스 URL을 입력하면 AI가 한국어로 번역하여 기사 형식으로 작성합니다.")
+        
+            with gr.Column():
+                chatbot = gr.Chatbot(height=600)
+            
+                with gr.Row():
+                    url_input = gr.Textbox(
+                        label="뉴스 URL",
+                        placeholder="https://..."
+                    )
+            
+                with gr.Accordion("고급 설정", open=False):
+                    system_message = gr.Textbox(
+                        value="You are a helpful translator that translates articles to Korean accurately and naturally. 1) '번역'을 먼저 출력하라. 2) 이어서 '번역된 내용'에 대해 '뉴스 기사' 형식으로 작성하여 신문 기사 양식으로 출력하라. 반드시 줄 바꾸기등 가독성 있게 출력하여야 하며, '습니다.'.'입니다.'등을 사용하지 말고 '다.'로 끝내야 한다. 철저하게 신문 기사 형식을 전문적인 기자가 작성하는 형식이어야 한다.",
+                        label="System message"
+                    )
+                    max_tokens = gr.Slider(
+                        minimum=1,
+                    maximum=4000,
+                        value=3824,
+                        step=1,
+                        label="Max new tokens"
+                    )
+                    temperature = gr.Slider(
+                        minimum=0.1,
+                        maximum=4.0,
+                        value=0.7,
+                        step=0.1,
+                        label="Temperature"
+                    )
+                    top_p = gr.Slider(
+                        minimum=0.1,
+                        maximum=1.0,
+                        value=0.95,
+                        step=0.05,
+                        label="Top-P"
+                    )
+            
+                translate_button = gr.Button("번역", variant="primary")
+            
+            # 이벤트 연결
+                translate_button.click(
+                    fn=respond,
+                    inputs=[
+                        url_input,
+                        chatbot,
+                        system_message,
+                        max_tokens,
+                        temperature,
+                        top_p,
+                    ],
+                    outputs=chatbot
+                )
+
+
+
+    
     # 이벤트 연결 부분
     # 국가별 탭 이벤트
     search_outputs = [status_message, translated_query_display, gr.Markdown(visible=False)]