from openai import OpenAI import anthropic from together import Together import json import re # Initialize clients anthropic_client = anthropic.Anthropic() openai_client = OpenAI() together_client = Together() SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "", "result": }. Ensure the output is valid JSON, without additional formatting or explanations.""" def get_openai_response(model_name, prompt): """Get response from OpenAI API""" try: response = openai_client.chat.completions.create( model=model_name, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt} ] ) return response.choices[0].message.content except Exception as e: return f"Error with OpenAI model {model_name}: {str(e)}" def get_anthropic_response(model_name, prompt): """Get response from Anthropic API""" try: response = anthropic_client.messages.create( model=model_name, max_tokens=1000, temperature=0, system=SYSTEM_PROMPT, messages=[ {"role": "user", "content": [{"type": "text", "text": prompt}]} ] ) return response.content[0].text except Exception as e: return f"Error with Anthropic model {model_name}: {str(e)}" def get_together_response(model_name, prompt): """Get response from Together API""" try: response = together_client.chat.completions.create( model=model_name, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt} ], stream=False ) return response.choices[0].message.content except Exception as e: return f"Error with Together model {model_name}: {str(e)}" def get_model_response(model_name, model_info, prompt): """Get response from appropriate API based on model organization""" if not model_info: return "Model not found or unsupported." api_model = model_info['api_model'] organization = model_info['organization'] try: if organization == 'OpenAI': return get_openai_response(api_model, prompt) elif organization == 'Anthropic': return get_anthropic_response(api_model, prompt) else: # All other organizations use Together API return get_together_response(api_model, prompt) except Exception as e: return f"Error with {organization} model {model_name}: {str(e)}" def parse_model_response(response): try: # Debug print print(f"Raw model response: {response}") # First try to parse the entire response as JSON try: data = json.loads(response) return str(data.get('result', 'N/A')), data.get('feedback', 'N/A') except json.JSONDecodeError: # If that fails (typically for smaller models), try to find JSON within the response json_match = re.search(r'{.*}', response) if json_match: data = json.loads(json_match.group(0)) return str(data.get('result', 'N/A')), data.get('feedback', 'N/A') else: return 'Error', f"Failed to parse response: {response}" except Exception as e: # Debug print for error case print(f"Failed to parse response: {str(e)}") return 'Error', f"Failed to parse response: {response}"