app.py

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from llama_cpp import Llama
from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
import uvicorn
from dotenv import load_dotenv
import io
import requests
import asyncio
import time

# Cargar variables de entorno
load_dotenv()

# Inicializar aplicación FastAPI
app = FastAPI()

# Configuración de los modelos
model_configs = [
    {"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf", "name": "GPT-2 XL"},
    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf", "name": "Meta Llama 3.1-8B Instruct"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf", "name": "Gemma 2-9B IT"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf", "name": "Gemma 2-27B"},
    {"repo_id": "Ffftdtd5dtft/Phi-3-mini-128k-instruct-Q2_K-GGUF", "filename": "phi-3-mini-128k-instruct-q2_k.gguf", "name": "Phi-3 Mini 128K Instruct"},
    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-q2_k.gguf", "name": "Meta Llama 3.1-8B"},
    {"repo_id": "Ffftdtd5dtft/Qwen2-7B-Instruct-Q2_K-GGUF", "filename": "qwen2-7b-instruct-q2_k.gguf", "name": "Qwen2 7B Instruct"},
    {"repo_id": "Ffftdtd5dtft/starcoder2-3b-Q2_K-GGUF", "filename": "starcoder2-3b-q2_k.gguf", "name": "Starcoder2 3B"},
    {"repo_id": "Ffftdtd5dtft/Qwen2-1.5B-Instruct-Q2_K-GGUF", "filename": "qwen2-1.5b-instruct-q2_k.gguf", "name": "Qwen2 1.5B Instruct"},
    {"repo_id": "Ffftdtd5dtft/starcoder2-15b-Q2_K-GGUF", "filename": "starcoder2-15b-q2_k.gguf", "name": "Starcoder2 15B"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-2b-it-Q2_K-GGUF", "filename": "gemma-2-2b-it-q2_k.gguf", "name": "Gemma 2-2B IT"},
    {"repo_id": "Ffftdtd5dtft/sarvam-2b-v0.5-Q2_K-GGUF", "filename": "sarvam-2b-v0.5-q2_k.gguf", "name": "Sarvam 2B v0.5"},
    {"repo_id": "Ffftdtd5dtft/WizardLM-13B-Uncensored-Q2_K-GGUF", "filename": "wizardlm-13b-uncensored-q2_k.gguf", "name": "WizardLM 13B Uncensored"},
    {"repo_id": "Ffftdtd5dtft/Qwen2-Math-72B-Instruct-Q2_K-GGUF", "filename": "qwen2-math-72b-instruct-q2_k.gguf", "name": "Qwen2 Math 72B Instruct"},
    {"repo_id": "Ffftdtd5dtft/WizardLM-7B-Uncensored-Q2_K-GGUF", "filename": "wizardlm-7b-uncensored-q2_k.gguf", "name": "WizardLM 7B Uncensored"},
    {"repo_id": "Ffftdtd5dtft/Qwen2-Math-7B-Instruct-Q2_K-GGUF", "filename": "qwen2-math-7b-instruct-q2_k.gguf", "name": "Qwen2 Math 7B Instruct"}
]

# Clase para gestionar modelos
class ModelManager:
    def __init__(self):
        self.models = []
        self.configs = {}

    async def download_model_to_memory(self, model_config):
        print(f"Descargando modelo: {model_config['name']}...")
        url = f"https://huggingface.co/{model_config['repo_id']}/resolve/main/{model_config['filename']}"
        response = requests.get(url)
        if response.status_code == 200:
            model_file = io.BytesIO(response.content)
            return model_file
        else:
            raise Exception(f"Error al descargar el modelo: {response.status_code}")

    async def load_model(self, model_config):
        try:
            start_time = time.time()
            model_file = await self.download_model_to_memory(model_config)
            print(f"Cargando modelo: {model_config['name']}...")
            
            # Cargar el modelo usando llama_cpp
            llama = await asyncio.get_event_loop().run_in_executor(
                None,
                lambda: Llama.from_pretrained(model_file)
            )
            
            # Simulación de división de carga si el tiempo excede 1 segundo
            async def load_part(part):
                # Esta función simula la carga de una parte del modelo
                await asyncio.sleep(0.1)  # Simula un pequeño retraso en la carga

            if time.time() - start_time > 1:
                print(f"Modelo {model_config['name']} tardó más de 1 segundo en cargarse, dividiendo la carga...")
                await asyncio.gather(*(load_part(part) for part in range(5)))  # Simulación de división en 5 partes
            
            tokenizer = llama.tokenizer

            # Almacenar tokens y tokenizer en la RAM
            model_data = {
                'model': llama,
                'tokenizer': tokenizer,
                'pad_token': tokenizer.pad_token,
                'pad_token_id': tokenizer.pad_token_id,
                'eos_token': tokenizer.eos_token,
                'eos_token_id': tokenizer.eos_token_id,
                'bos_token': tokenizer.bos_token,
                'bos_token_id': tokenizer.bos_token_id,
                'unk_token': tokenizer.unk_token,
                'unk_token_id': tokenizer.unk_token_id
            }
            
            self.models.append({"model_data": model_data, "name": model_config['name']})
        except Exception as e:
            print(f"Error al cargar el modelo: {e}")

    async def load_all_models(self):
        print("Iniciando carga de modelos...")
        start_time = time.time()
        tasks = [self.load_model(config) for config in model_configs]
        await asyncio.gather(*tasks)
        end_time = time.time()
        print(f"Todos los modelos han sido cargados en {end_time - start_time:.2f} segundos.")

# Instanciar ModelManager y cargar modelos
model_manager = ModelManager()

@app.on_event("startup")
async def startup_event():
    await model_manager.load_all_models()

# Modelo global para la solicitud de chat
class ChatRequest(BaseModel):
    message: str
    top_k: int = 50
    top_p: float = 0.95
    temperature: float = 0.7

# Límite de tokens para respuestas
TOKEN_LIMIT = 1000  # Define el límite de tokens permitido por respuesta

# Función para generar respuestas de chat
async def generate_chat_response(request, model_data):
    try:
        user_input = normalize_input(request.message)
        llama = model_data['model_data']['model']
        tokenizer = model_data['model_data']['tokenizer']
        
        # Generar respuesta de manera rápida
        response = await asyncio.get_event_loop().run_in_executor(
            None,
            lambda: llama.generate(user_input, max_length=TOKEN_LIMIT, do_sample=True, top_k=request.top_k, top_p=request.top_p, temperature=request.temperature)
        )
        generated_text = response['generated_text']
        # Dividir respuesta larga
        split_response = split_long_response(generated_text)
        return {"response": split_response, "literal": user_input, "model_name": model_data['name']}
    except Exception as e:
        print(f"Error al generar la respuesta: {e}")
        return {"response": "Error al generar la respuesta", "literal": user_input, "model_name": model_data['name']}

def split_long_response(response):
    """ Divide la respuesta en partes más pequeñas si excede el límite de tokens. """
    parts = []
    while len(response) > TOKEN_LIMIT:
        part = response[:TOKEN_LIMIT]
        response = response[TOKEN_LIMIT:]
        parts.append(part.strip())
    if response:
        parts.append(response.strip())
    return '\n'.join(parts)

def remove_duplicates(text):
    """ Elimina duplicados en el texto. """
    lines = text.splitlines()
    unique_lines = list(dict.fromkeys(lines))
    return '\n'.join(unique_lines)

def remove_repetitive_responses(responses):
    unique_responses = []
    seen_responses = set()
    for response in responses:
        normalized_response = remove_duplicates(response['response'])
        if normalized_response not in seen_responses:
            seen_responses.add(normalized_response)
            response['response'] = normalized_response
            unique_responses.append(response)
    return unique_responses

@app.post("/chat")
async def chat(request: ChatRequest):
    results = []
    for model_data in model_manager.models:
        response = await generate_chat_response(request, model_data)
        results.append(response)
    unique_results = remove_repetitive_responses(results)
    return {"results": unique_results}

# Ejecutar la aplicación FastAPI
if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8000)