Update app-backup2.py

app-backup2.py

@@ -3,12 +3,21 @@ import requests
 import json
 import os
 from datetime import datetime, timedelta
-from huggingface_hub import InferenceClient
-
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+from openai import OpenAI
 from bs4 import BeautifulSoup
-import concurrent.futures
-import time
-import re
+
+ACCESS_TOKEN = os.getenv("HF_TOKEN")
+if not ACCESS_TOKEN:
+    raise ValueError("HF_TOKEN environment variable is not set")
+
+client = OpenAI(
+    base_url="https://api-inference.huggingface.co/v1/",
+    api_key=ACCESS_TOKEN,
+)
 
 MAX_COUNTRY_RESULTS = 100    # 국가별 최대 결과 수
 MAX_GLOBAL_RESULTS = 1000    # 전세계 최대 결과 수
@@ -33,13 +42,12 @@ def create_article_components(max_results):
     return article_components
     
 API_KEY = os.getenv("SERPHOUSE_API_KEY")
-hf_client = InferenceClient("CohereForAI/c4ai-command-r-plus-08-2024", token=os.getenv("HF_TOKEN"))
 
 # 국가별 언어 코드 매핑
 COUNTRY_LANGUAGES = {
     "United States": "en",
     "United Kingdom": "en",
-    "Taiwan": "zh-TW",  # 대만어(번체 중국어)
+    "Taiwan": "zh-TW",  
     "Canada": "en",
     "Australia": "en",
     "Germany": "de",
@@ -109,7 +117,7 @@ COUNTRY_LANGUAGES = {
 COUNTRY_LOCATIONS = {
     "United States": "United States",
     "United Kingdom": "United Kingdom",
-    "Taiwan": "Taiwan",  # 국가명 사용
+    "Taiwan": "Taiwan",
     "Canada": "Canada",
     "Australia": "Australia",
     "Germany": "Germany",
@@ -176,8 +184,7 @@ COUNTRY_LOCATIONS = {
     "Iceland": "Iceland"
 }
 
-MAJOR_COUNTRIES = list(COUNTRY_LOCATIONS.keys())
-
+# 지역 정의
 # 동아시아 지역
 COUNTRY_LANGUAGES_EAST_ASIA = {
     "Taiwan": "zh-TW",  
@@ -352,24 +359,20 @@ REGIONS = [
     "아메리카"
 ]
 
+
+@lru_cache(maxsize=100)
 def translate_query(query, country):
     try:
-        # 영어 입력 확인
         if is_english(query):
-            print(f"영어 검색어 감지 - 원본 사용: {query}")
             return query
             
-        # 선택된 국가가 번역 지원 국가인 경우    
         if country in COUNTRY_LANGUAGES:
-            # South Korea 선택시 한글 입력은 그대로 사용    
             if country == "South Korea":
-                print(f"한국 선택 - 원본 사용: {query}")
                 return query
                 
             target_lang = COUNTRY_LANGUAGES[country]
-            print(f"번역 시도: {query} -> {country}({target_lang})")
             
-            url = f"https://translate.googleapis.com/translate_a/single"
+            url = "https://translate.googleapis.com/translate_a/single"
             params = {
                 "client": "gtx",
                 "sl": "auto",
@@ -378,9 +381,12 @@ def translate_query(query, country):
                 "q": query
             }
             
-            response = requests.get(url, params=params)
+            session = requests.Session()
+            retries = Retry(total=3, backoff_factor=0.5)
+            session.mount('https://', HTTPAdapter(max_retries=retries))
+            
+            response = session.get(url, params=params, timeout=(5, 10))
             translated_text = response.json()[0][0][0]
-            print(f"번역 완료: {query} -> {translated_text} ({country})")
             return translated_text
             
         return query
@@ -389,6 +395,8 @@ def translate_query(query, country):
         print(f"번역 오류: {str(e)}")
         return query
 
+
+@lru_cache(maxsize=200)
 def translate_to_korean(text):
     try:
         url = "https://translate.googleapis.com/translate_a/single"
@@ -400,7 +408,11 @@ def translate_to_korean(text):
             "q": text
         }
         
-        response = requests.get(url, params=params)
+        session = requests.Session()
+        retries = Retry(total=3, backoff_factor=0.5)
+        session.mount('https://', HTTPAdapter(max_retries=retries))
+        
+        response = session.get(url, params=params, timeout=(5, 10))
         translated_text = response.json()[0][0][0]
         return translated_text
     except Exception as e:
@@ -421,8 +433,6 @@ def search_serphouse(query, country, page=1, num_result=10):
     date_range = f"{yesterday.strftime('%Y-%m-%d')},{now.strftime('%Y-%m-%d')}"
     
     translated_query = translate_query(query, country)
-    print(f"Original query: {query}")
-    print(f"Translated query: {translated_query}")
     
     payload = {
         "data": {
@@ -446,14 +456,48 @@ def search_serphouse(query, country, page=1, num_result=10):
     }
 
     try:
-        response = requests.post(url, json=payload, headers=headers)
-        print("Request payload:", json.dumps(payload, indent=2, ensure_ascii=False))
-        print("Response status:", response.status_code)
+        # 세션 설정 개선
+        session = requests.Session()
+        
+        # 재시도 설정 강화
+        retries = Retry(
+            total=5,  # 최대 재시도 횟수 증가
+            backoff_factor=1,  # 재시도 간격 증가
+            status_forcelist=[500, 502, 503, 504, 429],  # 재시도할 HTTP 상태 코드
+            allowed_methods=["POST"]  # POST 요청에 대한 재시도 허용
+        )
+        
+        # 타임아웃 설정 조정
+        adapter = HTTPAdapter(max_retries=retries)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        
+        # 타임아웃 값 증가 (connect timeout, read timeout)
+        response = session.post(
+            url, 
+            json=payload, 
+            headers=headers, 
+            timeout=(30, 30)  # 연결 타임아웃 30초, 읽기 타임아웃 30초
+        )
         
         response.raise_for_status()
         return {"results": response.json(), "translated_query": translated_query}
-    except requests.RequestException as e:
-        return {"error": f"Error: {str(e)}", "translated_query": query}
+        
+    except requests.exceptions.Timeout:
+        return {
+            "error": "검색 시간이 초과되었습니다. 잠시 후 다시 시도해주세요.",
+            "translated_query": query
+        }
+    except requests.exceptions.RequestException as e:
+        return {
+            "error": f"검색 중 오류가 발생했습니다: {str(e)}",
+            "translated_query": query
+        }
+    except Exception as e:
+        return {
+            "error": f"예기치 않은 오류가 발생했습니다: {str(e)}",
+            "translated_query": query
+        }
 
 def format_results_from_raw(response_data):
     if "error" in response_data:
@@ -467,19 +511,35 @@ def format_results_from_raw(response_data):
         if not news_results:
             return "검색 결과가 없습니다.", []
 
-        articles = []
+        # 한국 도메인 및 한국 관련 키워드 필터링
+        korean_domains = ['.kr', 'korea', 'korean', 'yonhap', 'hankyung', 'chosun', 
+                         'donga', 'joins', 'hani', 'koreatimes', 'koreaherald']
+        korean_keywords = ['korea', 'korean', 'seoul', 'busan', 'incheon', 'daegu', 
+                         'gwangju', 'daejeon', 'ulsan', 'sejong']
+
+        filtered_articles = []
         for idx, result in enumerate(news_results, 1):
-            articles.append({
-                "index": idx,
-                "title": result.get("title", "제목 없음"),
-                "link": result.get("url", result.get("link", "#")),
-                "snippet": result.get("snippet", "내용 없음"),
-                "channel": result.get("channel", result.get("source", "알 수 없음")),
-                "time": result.get("time", result.get("date", "알 수 없는 시간")),
-                "image_url": result.get("img", result.get("thumbnail", "")),
-                "translated_query": translated_query
-            })
-        return "", articles
+            url = result.get("url", result.get("link", "")).lower()
+            title = result.get("title", "").lower()
+            channel = result.get("channel", result.get("source", "")).lower()
+            
+            # 한국 관련 컨텐츠 필터링
+            is_korean_content = any(domain in url or domain in channel for domain in korean_domains) or \
+                              any(keyword in title.lower() for keyword in korean_keywords)
+            
+            if not is_korean_content:
+                filtered_articles.append({
+                    "index": idx,
+                    "title": result.get("title", "제목 없음"),
+                    "link": url,
+                    "snippet": result.get("snippet", "내용 없음"),
+                    "channel": result.get("channel", result.get("source", "알 수 없음")),
+                    "time": result.get("time", result.get("date", "알 수 없는 시간")),
+                    "image_url": result.get("img", result.get("thumbnail", "")),
+                    "translated_query": translated_query
+                })
+
+        return "", filtered_articles
     except Exception as e:
         return f"결과 처리 중 오류 발생: {str(e)}", []
 
@@ -488,338 +548,174 @@ def serphouse_search(query, country):
     return format_results_from_raw(response_data)
 
 
-
-
-
-# Hacker News API 관련 함수들 먼저 추가
-def get_hn_item(item_id):
-    """개별 아이템 정보 가져오기"""
-    try:
-        response = requests.get(f"https://hacker-news.firebaseio.com/v0/item/{item_id}.json")
-        return response.json()
-    except:
-        return None
-
-def get_recent_stories():
-    """최신 스토리 가져오기"""
-    try:
-        response = requests.get("https://hacker-news.firebaseio.com/v0/newstories.json")
-        story_ids = response.json()
-        
-        recent_stories = []
-        current_time = datetime.now().timestamp()
-        day_ago = current_time - (24 * 60 * 60)
-        
-        for story_id in story_ids:
-            story = get_hn_item(story_id)
-            if story and 'time' in story and story['time'] > day_ago:
-                recent_stories.append(story)
-            
-            if len(recent_stories) >= 100:
-                break
-                
-        return recent_stories
-    except Exception as e:
-        print(f"Error fetching HN stories: {str(e)}")
-        return []
-
-def format_hn_time(timestamp):
-    """Unix timestamp를 읽기 쉬운 형식으로 변환"""
-    try:
-        dt = datetime.fromtimestamp(timestamp)
-        return dt.strftime("%Y-%m-%d %H:%M:%S")
-    except:
-        return "Unknown time"
-
-
-def clean_text(text):
-    """HTML 태그 제거 및 텍스트 정리"""
-    text = re.sub(r'\s+', ' ', text)
-    text = re.sub(r'<[^>]+>', '', text)
-    return text.strip()
-
-def get_article_content(url):
-    """URL에서 기사 내용 스크래핑"""
-    if not url:
-        return None
-        
-    # 스킵할 도메인 목록
-    skip_domains = ['github.com', 'twitter.com', 'linkedin.com', 'facebook.com']
-    if any(domain in url.lower() for domain in skip_domains):
-        return None
-        
-    try:
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-            'Connection': 'keep-alive',
-        }
-        
-        # 타임아웃 증가 및 재시도 설정
-        session = requests.Session()
-        retries = requests.adapters.Retry(total=3, backoff_factor=1)
-        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
+def search_and_display(query, country, articles_state, progress=gr.Progress()):
+    with ThreadPoolExecutor(max_workers=3) as executor:
+        progress(0, desc="검색어 번역 중...")
+        future_translation = executor.submit(translate_query, query, country)
+        translated_query = future_translation.result()
+        translated_display = f"**원본 검색어:** {query}\n**번역된 검색어:** {translated_query}" if translated_query != query else f"**검색어:** {query}"
         
-        response = session.get(url, headers=headers, timeout=15)
-        response.raise_for_status()
+        progress(0.3, desc="검색 중...")
+        response_data = search_serphouse(query, country)
         
-        soup = BeautifulSoup(response.text, 'html.parser')
+        progress(0.6, desc="결과 처리 중...")
+        error_message, articles = format_results_from_raw(response_data)
         
-        # 불필요한 요소 제거
-        for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside', 'iframe']):
-            tag.decompose()
-            
-        # 본문 내용 추출
-        article_text = ""
+        outputs = []
+        outputs.append(gr.update(value="검색을 진행중입니다...", visible=True))
+        outputs.append(gr.update(value=translated_display, visible=True))
         
-        # article 태그 확인
-        article = soup.find('article')
-        if article:
-            paragraphs = article.find_all('p')
+        if error_message:
+            outputs.append(gr.update(value=error_message, visible=True))
+            for comp in article_components:
+                outputs.extend([
+                    gr.update(visible=False), gr.update(), gr.update(),
+                    gr.update(), gr.update()
+                ])
+            articles_state = []
         else:
-            # main 태그 확인
-            main = soup.find('main')
-            if main:
-                paragraphs = main.find_all('p')
-            else:
-                # body에서 직접 검색
-                paragraphs = soup.find_all('p')
-        
-        text = ' '.join(p.get_text().strip() for p in paragraphs if p.get_text().strip())
-        text = clean_text(text)
-        
-        if not text:
-            return None
-            
-        return text[:4000]  # 텍스트 길이 제한
-        
-    except Exception as e:
-        print(f"Scraping error for {url}: {str(e)}")
-        return None
+            outputs.append(gr.update(value="", visible=False))
+            if not error_message and articles:
+                futures = []
+                for article in articles:
+                    future = executor.submit(translate_to_korean, article['snippet'])
+                    futures.append((article, future))
+                
+                progress(0.8, desc="번역 처리 중...")
+                for article, future in futures:
+                    article['korean_summary'] = future.result()
+                
+                total_articles = len(articles)
+                for idx, comp in enumerate(article_components):
+                    progress((idx + 1) / total_articles, desc=f"결과 표시 중... {idx + 1}/{total_articles}")
+                    if idx < len(articles):
+                        article = articles[idx]
+                        image_url = article['image_url']
+                        image_update = gr.update(value=image_url, visible=True) if image_url and not image_url.startswith('data:image') else gr.update(value=None, visible=False)
+                        
+                        outputs.extend([
+                            gr.update(visible=True),
+                            gr.update(value=f"### [{article['title']}]({article['link']})"),
+                            image_update,
+                            gr.update(value=f"**요약:** {article['snippet']}\n\n**한글 요약:** {article['korean_summary']}"),
+                            gr.update(value=f"**출처:** {article['channel']} | **시간:** {article['time']}")
+                        ])
+                    else:
+                        outputs.extend([
+                            gr.update(visible=False), gr.update(), gr.update(),
+                            gr.update(), gr.update()
+                        ])
+            articles_state = articles
 
-def generate_summary(text):
-    """CohereForAI 모델을 사용한 요약 생성"""
-    if not text:
-        return None
+        progress(1.0, desc="완료!")
+        outputs.append(articles_state)
+        outputs[0] = gr.update(value="", visible=False)
         
-    prompt = """반드시 한글(한국어)로 작성하라. Please analyze and summarize the following text in 2-3 sentences. 
-    Focus on the main points and key information:
+        return outputs
 
-    Text: {text}
-    
-    Summary:"""
-    
-    try:
-        response = hf_client.text_generation(
-            prompt.format(text=text),
-            max_new_tokens=500,
-            temperature=0.5,
-            repetition_penalty=1.2
-        )
-        return response
-    except Exception as e:
-        print(f"Summary generation error: {str(e)}")
-        return None
+def get_region_countries(region):
+    """선택된 지역의 국가 및 언어 정보 반환"""
+    if region == "동아시아":
+        return COUNTRY_LOCATIONS_EAST_ASIA, COUNTRY_LANGUAGES_EAST_ASIA
+    elif region == "동남아시아/오세아니아":
+        return COUNTRY_LOCATIONS_SOUTHEAST_ASIA_OCEANIA, COUNTRY_LANGUAGES_SOUTHEAST_ASIA_OCEANIA
+    elif region == "동유럽":
+        return COUNTRY_LOCATIONS_EAST_EUROPE, COUNTRY_LANGUAGES_EAST_EUROPE
+    elif region == "서유럽":
+        return COUNTRY_LOCATIONS_WEST_EUROPE, COUNTRY_LANGUAGES_WEST_EUROPE
+    elif region == "중동/아프리카":
+        return COUNTRY_LOCATIONS_ARAB_AFRICA, COUNTRY_LANGUAGES_ARAB_AFRICA
+    elif region == "아메리카":
+        return COUNTRY_LOCATIONS_AMERICA, COUNTRY_LANGUAGES_AMERICA
+    return {}, {}
 
-def process_hn_story(story, progress=None):
-    """개별 스토리 처리 및 요약"""
-    try:
-        url = story.get('url')
-        if not url:
-            return None  # 스킵할 스토리
-            
-        content = get_article_content(url)
-        if not content:
-            return None  # 스크래핑 실패한 스토리 스킵
-            
-        summary_en = generate_summary(content)
-        if not summary_en:
-            return None  # 요약 실패한 스토리 스킵
-            
-        summary_ko = translate_to_korean(summary_en)
-        if not summary_ko:
-            return None  # 번역 실패한 스토리 스킵
-            
-        return {
-            'story': story,
-            'summary': summary_ko
-        }
-        
-    except Exception as e:
-        print(f"Story processing error: {str(e)}")
-        return None  # 에러 발생한 스토리 스킵
+def search_global(query, region, articles_state_global):
+    """지역별 검색 함수"""
+    status_msg = f"{region} 지역 검색을 시작합니다..."
+    all_results = []
 
-def refresh_hn_stories():
-    """Hacker News 스토리 새로고침 (실시간 출력 버전)"""
-    status_msg = "Hacker News 포스트를 가져오는 중..."
-    outputs = [gr.update(value=status_msg, visible=True)]
+    outputs = [
+        gr.update(value=status_msg, visible=True),
+        gr.update(value=f"**검색어:** {query}", visible=True),
+    ]
     
-    # 컴포넌트 초기화
-    for comp in hn_article_components:
+    for _ in global_article_components:
         outputs.extend([
-            gr.update(visible=False),
-            gr.update(),
-            gr.update(),
-            gr.update(visible=False),  # report_button
-            gr.update(visible=False),  # report_content
-            gr.update(visible=False)   # show_report
+            gr.update(visible=False), gr.update(), gr.update(),
+            gr.update(), gr.update()
         ])
-    
+    outputs.append([])
+        
     yield outputs
     
-    # 최신 스토리 가져오기
-    stories = get_recent_stories()
-    processed_count = 0
-    valid_stories = []  # 성공적으로 처리된 스토리 저장
-    
-    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-        future_to_story = {executor.submit(process_hn_story, story): story 
-                          for story in stories[:100]}
-        
-        for future in concurrent.futures.as_completed(future_to_story):
-            processed_count += 1
-            result = future.result()
+    # 선택된 지역의 국가 정보 가져오기
+    locations, languages = get_region_countries(region)
+    total_countries = len(locations)
+
+    for idx, (country, location) in enumerate(locations.items(), 1):
+        try:
+            status_msg = f"{region} - {country} 검색 중... ({idx}/{total_countries} 국가)"
+            outputs[0] = gr.update(value=status_msg, visible=True)
+            yield outputs
             
-            if result:  # 성공적으로 처리된 스토리만 추가
-                valid_stories.append((result['story'], result['summary']))
+            error_message, articles = serphouse_search(query, country)
+            if not error_message and articles:
+                for article in articles:
+                    article['source_country'] = country
+                    article['region'] = region
                 
-                # 현재까지의 결과 출력
-                outputs = [gr.update(value=f"처리 중... ({len(valid_stories)}/{processed_count} 성공)", visible=True)]
+                all_results.extend(articles)
+                sorted_results = sorted(all_results, key=lambda x: x.get('time', ''), reverse=True)
                 
-                # 모든 컴포넌트 업데이트
-                for idx, comp in enumerate(hn_article_components):
-                    if idx < len(valid_stories):
-                        story, summary = valid_stories[idx]
+                seen_urls = set()
+                unique_results = []
+                for article in sorted_results:
+                    url = article.get('link', '')
+                    if url not in seen_urls:
+                        seen_urls.add(url)
+                        unique_results.append(article)
+            
+                unique_results = unique_results[:MAX_GLOBAL_RESULTS]
+            
+                outputs = [
+                    gr.update(value=f"{region} - {idx}/{total_countries} 국가 검색 완료\n현재까지 발견된 뉴스: {len(unique_results)}건", visible=True),
+                    gr.update(value=f"**검색어:** {query} | **지역:** {region}", visible=True),
+                ]
+            
+                for idx, comp in enumerate(global_article_components):
+                    if idx < len(unique_results):
+                        article = unique_results[idx]
+                        image_url = article.get('image_url', '')
+                        image_update = gr.update(value=image_url, visible=True) if image_url and not image_url.startswith('data:image') else gr.update(value=None, visible=False)
+                        
+                        korean_summary = translate_to_korean(article['snippet'])
+                        
                         outputs.extend([
                             gr.update(visible=True),
-                            gr.update(value=f"### [{story.get('title', 'Untitled')}]({story.get('url', '#')})"),
-                            gr.update(value=f"""
-                            **작성자:** {story.get('by', 'unknown')} | 
-                            **시간:** {format_hn_time(story.get('time', 0))} | 
-                            **점수:** {story.get('score', 0)} | 
-                            **댓글:** {len(story.get('kids', []))}개\n
-                            **AI 요약:** {summary}
-                            """),
-                            gr.update(visible=True),  # report_button
-                            gr.update(visible=False),  # report_content
-                            gr.update(visible=False)   # show_report
+                            gr.update(value=f"### [{article['title']}]({article['link']})"),
+                            image_update,
+                            gr.update(value=f"**요약:** {article['snippet']}\n\n**한글 요약:** {korean_summary}"),
+                            gr.update(value=f"**출처:** {article['channel']} | **국가:** {article['source_country']} | **지역:** {article['region']} | **시간:** {article['time']}")
                         ])
                     else:
                         outputs.extend([
                             gr.update(visible=False),
                             gr.update(),
                             gr.update(),
-                            gr.update(visible=False),
-                            gr.update(visible=False),
-                            gr.update(visible=False)
+                            gr.update(),
+                            gr.update()
                         ])
-                
-                yield outputs
-    
-    # 최종 상태 업데이트
-    final_outputs = [gr.update(value=f"총 {len(valid_stories)}개의 포스트가 성공적으로 처리되었습니다. (전체 시도: {processed_count})", visible=True)]
-    
-    for idx, comp in enumerate(hn_article_components):
-        if idx < len(valid_stories):
-            story, summary = valid_stories[idx]
-            final_outputs.extend([
-                gr.update(visible=True),
-                gr.update(value=f"### [{story.get('title', 'Untitled')}]({story.get('url', '#')})"),
-                gr.update(value=f"""
-                **작성자:** {story.get('by', 'unknown')} | 
-                **시간:** {format_hn_time(story.get('time', 0))} | 
-                **점수:** {story.get('score', 0)} | 
-                **댓글:** {len(story.get('kids', []))}개\n
-                **AI 요약:** {summary}
-                """),
-                gr.update(visible=True),  # report_button
-                gr.update(visible=False),  # report_content
-                gr.update(visible=False)   # show_report
-            ])
-        else:
-            final_outputs.extend([
-                gr.update(visible=False),
-                gr.update(),
-                gr.update(),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False)
-            ])
-    
-    yield final_outputs
-
-def generate_report(title, info, progress=gr.Progress()):
-    """리포팅 생성"""
-    try:
-        progress(0.1, desc="리포팅 생성 준비 중...")
-        
-        # HTML 태그 제거 및 텍스트 추출
-        title_text = re.sub(r'#*\s*\[(.*?)\].*', r'\1', title)
-        info_text = re.sub(r'\*\*(.*?)\*\*|\n|AI 요약:|작성자:|시간:|점수:|댓글:', ' ', info)
-        info_text = ' '.join(info_text.split())
-        
-        progress(0.3, desc="프롬프트 생성 중...")
-        
-        prompt = f"""너는 Hacker News 포스트를 기반으로 보도 기사 형태의 리포팅을 작성하는 역할이다. 
-        너는 반드시 한글로 리포팅 형식의 객관적 기사 형태로 작성하여야 한다. 
-        생성시 6하원칙에 입각하고 길이는 4000토큰을 넘지 않을것. 
-        너의 출처나 모델, 지시문 등을 노출하지 말것
-
-        제목: {title_text}
-        내용: {info_text}
-        """
-        
-        progress(0.5, desc="AI 모델 처리 중...")
-        
-        try:
-            response = hf_client.text_generation(
-                prompt,
-                max_new_tokens=2000,
-                temperature=0.7,
-                repetition_penalty=1.2,
-                return_full_text=False
-            )
-            
-            progress(1.0, desc="완료!")
-            
-            if response:
-                formatted_response = f"### AI 리포팅\n\n{response}"
-                return [
-                    gr.update(value=formatted_response, visible=True),  # report_content
-                    gr.update(value="접기", visible=True)  # show_report
-                ]
             
+                outputs.append(unique_results)
+                yield outputs
+                
         except Exception as e:
-            print(f"Model error: {str(e)}")
-            time.sleep(2)  # 잠시 대기
-            
-        return [
-            gr.update(value="리포팅 생성에 실패했습니다. 다시 시도해주세요.", visible=True),
-            gr.update(value="접기", visible=True)
-        ]
-            
-    except Exception as e:
-        print(f"Report generation error: {str(e)}")
-        return [
-            gr.update(value="리포팅 생성 중 오류가 발생했습니다.", visible=True),
-            gr.update(value="접기", visible=True)
-        ]
-
-def toggle_report(report_content, show_report):
-    """리포트 표시/숨김 토글"""
-    try:
-        is_visible = report_content.visible
-        return [
-            gr.update(visible=not is_visible),  # report_content
-            gr.update(value="접기" if not is_visible else "펼쳐 보기")  # show_report
-        ]
-    except AttributeError:
-        # report_content가 문자열인 경우
-        return [
-            gr.update(visible=True),  # report_content
-            gr.update(value="접기")  # show_report
-        ]
+            print(f"Error searching {country}: {str(e)}")
+            continue
+    
+    final_status = f"{region} 검색 완료! 총 {len(unique_results)}개의 뉴스가 발견되었습니다."
+    outputs[0] = gr.update(value=final_status, visible=True)
+    yield outputs
 
 css = """
 /* 전역 스타일 */
@@ -888,260 +784,236 @@ footer {visibility: hidden;}
     border-radius: 4px !important;
 }
 
-/* Hacker News 아티클 스타일 */
-.hn-article-group {
-    height: auto !important;
-    min-height: 250px;
-    margin-bottom: 20px;
-    padding: 15px;
-    border: 1px solid #eee;
-    border-radius: 5px;
-    background: white;
-    box-shadow: 0 1px 3px rgba(0,0,0,0.05);
-}
-
-/* 리포트 섹션 스타일 */
-.report-section {
-    margin-top: 15px;
-    padding: 15px;
-    border-top: 1px solid #eee;
-    background: #f9f9f9;
-    border-radius: 4px;
-}
-
-.report-content {
-    margin-top: 15px;
-    padding: 15px;
-    border-top: 1px solid #eee;
-    background: #f9f9f9;
-    border-radius: 4px;
-    font-size: 0.95em;
-    line-height: 1.6;
-}
-
-/* 프로그레스 바 */
-.progress {
+/* 프로그레스바 컨테이너 */
+.progress-container {
     position: fixed;
     top: 0;
     left: 0;
     width: 100%;
-    height: 4px;
-    background: #f0f0f0;
+    height: 6px;
+    background: #e0e0e0;
     z-index: 1000;
 }
 
+/* 프로그레스바 */
 .progress-bar {
     height: 100%;
-    background: #1f77b4;
+    background: linear-gradient(90deg, #2196F3, #00BCD4);
+    box-shadow: 0 0 10px rgba(33, 150, 243, 0.5);
     transition: width 0.3s ease;
-    position: fixed;
-    top: 0;
-    left: 0;
-    width: 100%;
-    z-index: 1000;
+    animation: progress-glow 1.5s ease-in-out infinite;
 }
 
-/* 리포트 콘텐츠 토글 */
-.hn-article-group .report-content {
-    display: none;
-    margin-top: 15px;
-    padding: 15px;
-    border-top: 1px solid #eee;
-    background: #f9f9f9;
-    transition: all 0.3s ease;
+/* 프로그레스 텍스트 */
+.progress-text {
+    position: fixed;
+    top: 8px;
+    left: 50%;
+    transform: translateX(-50%);
+    background: #333;
+    color: white;
+    padding: 4px 12px;
+    border-radius: 15px;
+    font-size: 14px;
+    z-index: 1001;
+    box-shadow: 0 2px 5px rgba(0,0,0,0.2);
 }
 
-.hn-article-group .report-content.visible {
-    display: block;
+/* 프로그레스바 애니메이션 */
+@keyframes progress-glow {
+    0% {
+        box-shadow: 0 0 5px rgba(33, 150, 243, 0.5);
+    }
+    50% {
+        box-shadow: 0 0 20px rgba(33, 150, 243, 0.8);
+    }
+    100% {
+        box-shadow: 0 0 5px rgba(33, 150, 243, 0.5);
+    }
 }
 
 /* 반응형 디자인 */
 @media (max-width: 768px) {
-    .hn-article-group {
+    .group {
         padding: 10px;
         margin-bottom: 15px;
     }
     
-    .report-content {
-        padding: 10px;
+    .progress-text {
+        font-size: 12px;
+        padding: 3px 10px;
     }
 }
-"""
 
+/* 로딩 상태 표시 개선 */
+.loading {
+    opacity: 0.7;
+    pointer-events: none;
+    transition: opacity 0.3s ease;
+}
 
+/* 결과 컨테이너 애니메이션 */
+.group {
+    transition: all 0.3s ease;
+    opacity: 0;
+    transform: translateY(20px);
+}
 
- 
-    
-# 기존 함수들
-def search_and_display(query, country, articles_state, progress=gr.Progress()):
-    status_msg = "검색을 진행중입니다. 잠시만 기다리세요..."
-    
-    progress(0, desc="검색어 번역 중...")
-    translated_query = translate_query(query, country)
-    translated_display = f"**원본 검색어:** {query}\n**번역된 검색어:** {translated_query}" if translated_query != query else f"**검색어:** {query}"
-    
-    progress(0.2, desc="검색 시작...")
-    error_message, articles = serphouse_search(query, country)
-    progress(0.5, desc="결과 처리 중...")
-    
-    outputs = []
-    outputs.append(gr.update(value=status_msg, visible=True))
-    outputs.append(gr.update(value=translated_display, visible=True))
-    
-    if error_message:
-        outputs.append(gr.update(value=error_message, visible=True))
-        for comp in article_components:
-            outputs.extend([
-                gr.update(visible=False), gr.update(), gr.update(),
-                gr.update(), gr.update()
-            ])
-        articles_state = []
-    else:
-        outputs.append(gr.update(value="", visible=False))
-        total_articles = len(articles)
-        for idx, comp in enumerate(article_components):
-            progress((idx + 1) / total_articles, desc=f"결과 표시 중... {idx + 1}/{total_articles}")
-            if idx < len(articles):
-                article = articles[idx]
-                image_url = article['image_url']
-                image_update = gr.update(value=image_url, visible=True) if image_url and not image_url.startswith('data:image') else gr.update(value=None, visible=False)
-                
-                korean_summary = translate_to_korean(article['snippet'])
-                
-                outputs.extend([
-                    gr.update(visible=True),
-                    gr.update(value=f"### [{article['title']}]({article['link']})"),
-                    image_update,
-                    gr.update(value=f"**요약:** {article['snippet']}\n\n**한글 요약:** {korean_summary}"),
-                    gr.update(value=f"**출처:** {article['channel']} | **시간:** {article['time']}")
-                ])
-            else:
-                outputs.extend([
-                    gr.update(visible=False), gr.update(), gr.update(),
-                    gr.update(), gr.update()
-                ])
-        articles_state = articles
+.group.visible {
+    opacity: 1;
+    transform: translateY(0);
+}
 
-    progress(1.0, desc="완료!")
-    outputs.append(articles_state)
-    outputs[0] = gr.update(value="", visible=False)
-    
-    return outputs
+/* Examples 스타일링 */
+.examples-table {
+    margin-top: 10px !important;
+    margin-bottom: 20px !important;
+}
 
+.examples-table button {
+    background-color: #f0f0f0 !important;
+    border: 1px solid #ddd !important;
+    border-radius: 4px !important;
+    padding: 5px 10px !important;
+    margin: 2px !important;
+    transition: all 0.3s ease !important;
+}
 
-def get_region_countries(region):
-    """선택된 지역의 국가 및 언어 정보 반환"""
-    if region == "동아시아":
-        return COUNTRY_LOCATIONS_EAST_ASIA, COUNTRY_LANGUAGES_EAST_ASIA
-    elif region == "동남아시아/오세아니아":
-        return COUNTRY_LOCATIONS_SOUTHEAST_ASIA_OCEANIA, COUNTRY_LANGUAGES_SOUTHEAST_ASIA_OCEANIA
-    elif region == "동유럽":
-        return COUNTRY_LOCATIONS_EAST_EUROPE, COUNTRY_LANGUAGES_EAST_EUROPE
-    elif region == "서유럽":
-        return COUNTRY_LOCATIONS_WEST_EUROPE, COUNTRY_LANGUAGES_WEST_EUROPE
-    elif region == "중동/아프리카":
-        return COUNTRY_LOCATIONS_ARAB_AFRICA, COUNTRY_LANGUAGES_ARAB_AFRICA
-    elif region == "아메리카":
-        return COUNTRY_LOCATIONS_AMERICA, COUNTRY_LANGUAGES_AMERICA
-    return {}, {}
+.examples-table button:hover {
+    background-color: #e0e0e0 !important;
+    transform: translateY(-1px) !important;
+    box-shadow: 0 2px 5px rgba(0,0,0,0.1) !important;
+}
 
-def search_global(query, region, articles_state_global):
-    """지역별 검색 함수"""
-    status_msg = f"{region} 지역 검색을 시작합니다..."
-    all_results = []
+.examples-table .label {
+    font-weight: bold !important;
+    color: #444 !important;
+    margin-bottom: 5px !important;
+}
+"""
 
-    outputs = [
-        gr.update(value=status_msg, visible=True),
-        gr.update(value=f"**검색어:** {query}", visible=True),
-    ]
-    
-    for _ in global_article_components:
-        outputs.extend([
-            gr.update(visible=False), gr.update(), gr.update(),
-            gr.update(), gr.update()
-        ])
-    outputs.append([])
-        
-    yield outputs
-    
-    # 선택된 지역의 국가 정보 가져오기
-    locations, languages = get_region_countries(region)
-    total_countries = len(locations)
 
-    for idx, (country, location) in enumerate(locations.items(), 1):
-        try:
-            status_msg = f"{region} - {country} 검색 중... ({idx}/{total_countries} 국가)"
-            outputs[0] = gr.update(value=status_msg, visible=True)
-            yield outputs
-            
-            error_message, articles = serphouse_search(query, country)
-            if not error_message and articles:
-                for article in articles:
-                    article['source_country'] = country
-                    article['region'] = region
-                
-                all_results.extend(articles)
-                sorted_results = sorted(all_results, key=lambda x: x.get('time', ''), reverse=True)
-                
-                seen_urls = set()
-                unique_results = []
-                for article in sorted_results:
-                    url = article.get('link', '')
-                    if url not in seen_urls:
-                        seen_urls.add(url)
-                        unique_results.append(article)
-            
-                unique_results = unique_results[:MAX_GLOBAL_RESULTS]
-            
-                outputs = [
-                    gr.update(value=f"{region} - {idx}/{total_countries} 국가 검색 완료\n현재까지 발견된 뉴스: {len(unique_results)}건", visible=True),
-                    gr.update(value=f"**검색어:** {query} | **지역:** {region}", visible=True),
-                ]
-            
-                for idx, comp in enumerate(global_article_components):
-                    if idx < len(unique_results):
-                        article = unique_results[idx]
-                        image_url = article.get('image_url', '')
-                        image_update = gr.update(value=image_url, visible=True) if image_url and not image_url.startswith('data:image') else gr.update(value=None, visible=False)
-                        
-                        korean_summary = translate_to_korean(article['snippet'])
-                        
-                        outputs.extend([
-                            gr.update(visible=True),
-                            gr.update(value=f"### [{article['title']}]({article['link']})"),
-                            image_update,
-                            gr.update(value=f"**요약:** {article['snippet']}\n\n**한글 요약:** {korean_summary}"),
-                            gr.update(value=f"**출처:** {article['channel']} | **국가:** {article['source_country']} | **지역:** {article['region']} | **시간:** {article['time']}")
-                        ])
-                    else:
-                        outputs.extend([
-                            gr.update(visible=False),
-                            gr.update(),
-                            gr.update(),
-                            gr.update(),
-                            gr.update()
-                        ])
+def get_article_content(url):
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        response = requests.get(url, headers=headers)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # 일반적인 기사 본문 컨테이너 검색
+        article_body = None
+        possible_content_elements = [
+            soup.find('article'),
+            soup.find('div', class_='article-body'),
+            soup.find('div', class_='content'),
+            soup.find('div', {'id': 'article-body'})
+        ]
+        
+        for element in possible_content_elements:
+            if element:
+                article_body = element
+                break
+        
+        if article_body:
+            # 불필요한 요소 제거
+            for tag in article_body.find_all(['script', 'style', 'nav', 'header', 'footer']):
+                tag.decompose()
             
-                outputs.append(unique_results)
-                yield outputs
-                
-        except Exception as e:
-            print(f"Error searching {country}: {str(e)}")
-            continue
-    
-    final_status = f"{region} 검색 완료! 총 {len(unique_results)}개의 뉴스가 발견되었습니다."
-    outputs[0] = gr.update(value=final_status, visible=True)
-    yield outputs
-
+            content = ' '.join([p.get_text().strip() for p in article_body.find_all('p') if p.get_text().strip()])
+        else:
+            content = ' '.join([p.get_text().strip() for p in soup.find_all('p') if p.get_text().strip()])
+        
+        return content
+    except Exception as e:
+        return f"Error crawling content: {str(e)}"
+
+def respond(
+    url,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    if not url.startswith('http'):
+        history.append((url, "올바른 URL을 입력해주세요."))
+        return history
 
-    
-  
+    try:
+        # 기사 내용 추출
+        article_content = get_article_content(url)
+        
+        # 2단계 프로세스를 위한 프롬프트 구성
+        translation_prompt = f"""다음 작업을 순차적으로 수행하세요:
+
+        1단계: 번역
+        아래 영문 기사를 한국어로 정확하게 번역하세요.
+        구분선: ===번역 시작===
+        {article_content}
+        구분선: ===번역 끝===
+
+        2단계: 기사 작성
+        위의 번역된 내용을 바탕으로 새로운 한국어 기사를 작성하세요.
+        다음 형식을 반드시 준수하세요:
+        - 제목: [헤드라인]
+        - 부제: [서브헤드라인]
+        - 본문: [기사 내용]
+        - 작성 규칙:
+          * 문장은 '다.'로 끝나야 함
+          * 신문 기사 형식 준수
+          * 단락 구분을 명확히 할 것
+          * 핵심 정보를 앞부분에 배치
+          * 인용구는 따옴표로 처리
+
+        각 단계는 '===번역===', '===기사==='로 구분하여 출력하세요.
+        """
 
+        messages = [
+            {
+                "role": "system", 
+                "content": """당신은 전문 번역가이자 기자입니다. 
+                모든 작업은 반드시 다음 두 단계로 진행하고, 각 단계를 명확히 구분하여 출력해야 합니다:
+                1. 원문 번역: ===번역=== 표시 후 정확한 한국어 번역 제공
+                2. 기사 작성: ===기사=== 표시 후 번역본을 기반으로 한국어 뉴스 기사 작성
+                두 단계를 건너뛰거나 통합하지 말고 반드시 순차적으로 진행하세요."""
+            },
+            {"role": "user", "content": translation_prompt}
+        ]
 
+        history.append((url, "번역 및 기사 작성을 시작합니다..."))
+        
+        full_response = ""
+        current_section = ""
+        
+        for message in client.chat.completions.create(
+            model="CohereForAI/c4ai-command-r-plus-08-2024",
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
+            messages=messages,
+        ):
+            if hasattr(message.choices[0].delta, 'content'):
+                token = message.choices[0].delta.content
+                if token:
+                    full_response += token
+                    # 섹션 구분자 확인 및 포맷팅
+                    if "===번역===" in token or "===기사===" in token:
+                        current_section = token.strip()
+                        full_response += "\n\n"
+                    
+                    history[-1] = (url, full_response)
+                    yield history
 
+    except Exception as e:
+        error_message = f"처리 중 오류가 발생했습니다: {str(e)}"
+        history.append((url, error_message))
+        yield history
 
+    return history
 
-with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as iface:
+with gr.Blocks(theme="Yntec/HaleyCH_Theme_Orange", css=css, title="NewsAI 서비스") as iface:
     with gr.Tabs():
         # 국가별 탭
         with gr.Tab("국가별"):
@@ -1157,9 +1029,35 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
                         value="United States"
                     )
                 
+                # Examples 추가
+                gr.Examples(
+                    examples=[
+                        "artificial intelligence",
+                        "NVIDIA",
+                        "OPENAI",
+                        "META LLAMA",
+                        "black forest labs",
+                        "GOOGLE gemini",
+                        "anthropic Claude",
+                        "X.AI",
+                        "HUGGINGFACE",
+                        "HYNIX",
+                        "Large Language model",
+                        "CHATGPT",
+                        "StabilityAI",
+                        "MISTRALAI",
+                        "QWEN",                        
+                        "MIDJOURNEY",
+                        "GPU"
+                    ],
+                    inputs=query,
+                    label="자주 사용되는 검색어"
+                )
+                
                 status_message = gr.Markdown("", visible=True)
                 translated_query_display = gr.Markdown(visible=False)
                 search_button = gr.Button("검색", variant="primary")
+            
                 
                 progress = gr.Progress()
                 articles_state = gr.State([])
@@ -1218,36 +1116,87 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
                                 'index': i,
                             })
 
-        # AI 리포터 탭
-        with gr.Tab("AI 리포터"):
-            gr.Markdown("지난 24시간 동안의 Hacker News 포스트를 AI가 요약하여 보여줍니다.")
-            
+    
+    # AI 번역 탭 추가
+        with gr.Tab("AI 기사 생성"):
+            gr.Markdown("뉴스 URL을 입력하면 AI가 한국어로 번역하여 기사 형식으로 작성합니다.")
+        
             with gr.Column():
-                refresh_button = gr.Button("새로고침", variant="primary")
-                status_message_hn = gr.Markdown("")
-                
-                with gr.Column(elem_id="hn_results_area"):
-                    hn_articles_state = gr.State([])
-                    hn_article_components = []
-                    for i in range(100):
-                        with gr.Group(visible=False, elem_classes="hn-article-group") as article_group:
-                            title = gr.Markdown()
-                            info = gr.Markdown()
-                            with gr.Row():
-                                report_button = gr.Button("리포팅 생성", size="sm", variant="primary")
-                                show_report = gr.Button("펼쳐 보기", size="sm", visible=False)
-                            report_content = gr.Markdown(visible=False)
-                            
-                            hn_article_components.append({
-                                'group': article_group,
-                                'title': title,
-                                'info': info,
-                                'report_button': report_button,
-                                'show_report': show_report,
-                                'report_content': report_content,
-                                'index': i,
-                            })
+                chatbot = gr.Chatbot(height=600)
+            
+                with gr.Row():
+                    url_input = gr.Textbox(
+                        label="뉴스 URL",
+                        placeholder="https://..."
+                    )
+            
+                with gr.Accordion("고급 설정", open=False):
+                    system_message = gr.Textbox(
+                        value="""You are a professional translator and journalist. Follow these steps strictly:
+                        1. TRANSLATION
+                        - Start with ===번역=== marker
+                        - Provide accurate Korean translation
+                        - Maintain original meaning and context
+    
+                        2. ARTICLE WRITING
+                        - Start with ===기사=== marker
+                        - Write a new Korean news article based on the translation
+                        - Follow newspaper article format
+                        - Use formal news writing style
+                        - End sentences with '다.'
+                        - Include headline and subheadline
+                        - Organize paragraphs clearly
+                        - Put key information first
+                        - Use quotes appropriately
+        
+                        IMPORTANT:
+                        - Must complete both steps in order
+                        - Clearly separate each section with markers
+                        - Never skip or combine steps""",
+                        label="System message"
+                    )
+
+                    max_tokens = gr.Slider(
+                        minimum=1,
+                    maximum=7800,
+                        value=7624,
+                        step=1,
+                        label="Max new tokens"
+                    )
+                    temperature = gr.Slider(
+                        minimum=0.1,
+                        maximum=4.0,
+                        value=0.7,
+                        step=0.1,
+                        label="Temperature"
+                    )
+                    top_p = gr.Slider(
+                        minimum=0.1,
+                        maximum=1.0,
+                        value=0.95,
+                        step=0.05,
+                        label="Top-P"
+                    )
+            
+                translate_button = gr.Button("기사 생성", variant="primary")
+            
+            # 이벤트 연결
+                translate_button.click(
+                    fn=respond,
+                    inputs=[
+                        url_input,
+                        chatbot,
+                        system_message,
+                        max_tokens,
+                        temperature,
+                        top_p,
+                    ],
+                    outputs=chatbot
+                )
+
 
+
+    
     # 이벤트 연결 부분
     # 국가별 탭 이벤트
     search_outputs = [status_message, translated_query_display, gr.Markdown(visible=False)]
@@ -1281,61 +1230,11 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
         show_progress=True
     )
 
-    # AI 리포터 탭 이벤트
-    hn_outputs = [status_message_hn]
-    for comp in hn_article_components:
-        hn_outputs.extend([
-            comp['group'],
-            comp['title'],
-            comp['info'],
-            comp['report_button'],
-            comp['report_content'],
-            comp['show_report']
-        ])
-
-    # 각 컴포넌트별 이벤트 연결
-    for comp in hn_article_components:
-        # 리포팅 생성 버튼 이벤트
-        comp['report_button'].click(
-            fn=generate_report,
-            inputs=[
-                comp['title'],
-                comp['info']
-            ],
-            outputs=[
-                comp['report_content'],
-                comp['show_report']
-            ],
-            api_name=f"generate_report_{comp['index']}",
-            show_progress=True
-        )
-        
-        # 펼쳐보기/접기 버튼 이벤트
-        comp['show_report'].click(
-            fn=toggle_report,
-            inputs=[
-                comp['report_content'],
-                comp['show_report']
-            ],
-            outputs=[
-                comp['report_content'],
-                comp['show_report']
-            ],
-            api_name=f"toggle_report_{comp['index']}"
-        )
-
-    # 새로고침 버튼 이벤트
-    refresh_button.click(
-        fn=refresh_hn_stories,
-        outputs=hn_outputs,
-        show_progress=True
-    )
-
 iface.launch(
     server_name="0.0.0.0",
     server_port=7860,
     share=True,
-    auth=("it1","chosun1"),
+    auth=("ai","news"),
     ssl_verify=False,
     show_error=True
-)
+)