fix: deployment

README.md

@@ -7,5 +7,6 @@ sdk: gradio
 sdk_version: 4.44.1
 app_file: app.py
 pinned: false
+license: apache-2.0
 short_description: demo
 ---

app.py

@@ -289,7 +289,7 @@ with gr.Blocks(
             with Modal(visible=False, elem_classes="modal") as modal:
                 prompt = gr.Markdown("")
 
-    ### events
+    # events
 
     show_propt_button.click(
         on_show_prompt_click, inputs=[criteria, context, user_message, assistant_message, state], outputs=prompt

model.py

@@ -6,7 +6,7 @@ import spaces
 import torch
 from ibm_watsonx_ai.client import APIClient
 from ibm_watsonx_ai.foundation_models import ModelInference
-from transformers import AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from vllm import LLM, SamplingParams
 
 from logger import logger
@@ -24,7 +24,8 @@ if inference_engine == "VLLM":
     logger.debug(f"model_path is {model_path}")
     tokenizer = AutoTokenizer.from_pretrained(model_path)
     sampling_params = SamplingParams(temperature=0.0, logprobs=nlogprobs)
-    model = LLM(model=model_path, tensor_parallel_size=1)
+    # model = LLM(model=model_path, tensor_parallel_size=1)
+    model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16, device_map="auto")
 
 elif inference_engine == "WATSONX":
     client = APIClient(

utils.py

@@ -29,13 +29,13 @@ def get_result_description(sub_catalog_name, criteria_name):
     messages = {
         "general_harm": f"Is the {evaluated_component} message harmful by common-sense?",
         "social_bias": f"Is the {evaluated_component} message prejudiced based on identity or characteristics?",
-        "jailbreak": f"Does the user message try to manipulate the AI to generate harmful, undesired, or inappropriate content?",
+        "jailbreak": "Does the user message try to manipulate the AI to generate harmful, undesired, or inappropriate content?",
         "violence": f"Does the {evaluated_component} message contain content promoting physical, mental, or sexual harm?",
         "profanity": f"Does the {evaluated_component} message include offensive language or insults?",
         "unethical_behavior": f"Does the {evaluated_component} message include actions that violate moral or legal standards?",
-        "answer_relevance": f"Does the assistant response fail to address or properly answer the user question?",
-        "context_relevance": f"Is the retrieved context irrelevant to the user question or does not address their needs?",
-        "groundedness": f"Does the assistant response include claims or facts not supported by or contradicted by the provided context?",
+        "answer_relevance": "Does the assistant response fail to address or properly answer the user question?",
+        "context_relevance": "Is the retrieved context irrelevant to the user question or does not address their needs?",
+        "groundedness": "Does the assistant response include claims or facts not supported by or contradicted by the provided context?",
     }
     return messages[criteria_name]