from gradio.components import Component
import gradio as gr
from abc import ABC, abstractclassmethod
import inspect
class BaseTCOModel(ABC):
# TO DO: Find way to specify which component should be used for computing cost
def __setattr__(self, name, value):
if isinstance(value, Component):
self._components.append(value)
self.__dict__[name] = value
def __init__(self):
super(BaseTCOModel, self).__setattr__("_components", [])
def get_components(self) -> list[Component]:
return self._components
def get_components_for_cost_computing(self):
return self.components_for_cost_computing
def get_name(self):
return self.name
def register_components_for_cost_computing(self):
args = inspect.getfullargspec(self.compute_cost_per_token)[0][1:]
self.components_for_cost_computing = [self.__getattribute__(arg) for arg in args]
@abstractclassmethod
def compute_cost_per_token(self):
pass
@abstractclassmethod
def render(self):
pass
def set_name(self, name):
self.name = name
def set_formula(self, formula):
self.formula = formula
def get_formula(self):
return self.formula
class OpenAIModel(BaseTCOModel):
def __init__(self):
self.set_name("(SaaS) OpenAI")
self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$
with:
CT = Cost per output Token
CT_1K = Cost per 1000 Tokens (from OpenAI's pricing web page)
L = Input Length
""")
super().__init__()
def render(self):
def on_model_change(model):
if model == "GPT-4":
return gr.Dropdown.update(choices=["8K", "32K"])
else:
return gr.Dropdown.update(choices=["4K", "16K"])
self.model = gr.Dropdown(["GPT-4", "GPT-3.5 Turbo"], value="GPT-4",
label="OpenAI models",
interactive=True, visible=False)
self.context_length = gr.Dropdown(["8K", "32K"], value="8K", interactive=True,
label="Context size",
visible=False, info="Number of tokens the model considers when processing text")
self.model.change(on_model_change, inputs=self.model, outputs=self.context_length)
self.input_length = gr.Number(350, label="Average number of input tokens",
interactive=True, visible=False)
def compute_cost_per_token(self, model, context_length, input_length):
"""Cost per token = """
model = model[0]
context_length = context_length[0]
if model == "GPT-4" and context_length == "8K":
cost_per_1k_input_tokens = 0.03
elif model == "GPT-4" and context_length == "32K":
cost_per_1k_input_tokens = 0.06
elif model == "GPT-3.5" and context_length == "4K":
cost_per_1k_input_tokens = 0.0015
else:
cost_per_1k_input_tokens = 0.003
cost_per_output_token = cost_per_1k_input_tokens * input_length / 1000
return cost_per_output_token
class OpenSourceLlama2Model(BaseTCOModel):
def __init__(self):
self.set_name("(Open source) Llama 2")
self.set_formula(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$
with:
CT = Cost per Token
VM_CH = VM Cost per Hour
TS = Tokens per Second (for an input length of 233 tokens)
MO = Maxed Out
U = Used
""")
super().__init__()
def render(self):
vm_choices = ["1x Nvidia A100 (Azure NC24ads A100 v4)",
"2x Nvidia A100 (Azure NC48ads A100 v4)"]
def on_model_change(model):
if model == "Llama 2 7B":
return gr.Dropdown.update(choices=vm_choices)
else:
not_supported_vm = ["1x Nvidia A100 (Azure NC24ads A100 v4)"]
choices = [x for x in vm_choices if x not in not_supported_vm]
return gr.Dropdown.update(choices=choices)
def on_vm_change(model, vm):
# TO DO: load info from CSV
if model == "Llama 2 7B" and vm == "1x Nvidia A100 (Azure NC24ads A100 v4)":
return [gr.Number.update(value=3.6730), gr.Number.update(value=694.38)]
elif model == "Llama 2 7B" and vm == "2x Nvidia A100 (Azure NC48ads A100 v4)":
return [gr.Number.update(value=7.346), gr.Number.update(value=1388.76)]
self.model = gr.Dropdown(["Llama 2 7B", "Llama 2 70B"], value="Llama 2 7B", label="OpenSource models", visible=False)
self.vm = gr.Dropdown(vm_choices,
value="1x Nvidia A100 (Azure NC24ads A100 v4)",
visible=False,
label="Instance of VM with GPU",
info="Your options for this choice depend on the model you previously chose"
)
self.vm_cost_per_hour = gr.Number(3.6730, label="VM instance cost per hour",
interactive=True, visible=False)
self.tokens_per_second = gr.Number(694.38, visible=False,
label="Number of tokens per second for this specific model and VM instance",
interactive=False
)
self.input_length = gr.Number(350, label="Average number of input tokens",
interactive=True, visible=False)
self.model.change(on_model_change, inputs=self.model, outputs=self.vm)
self.vm.change(on_vm_change, inputs=[self.model, self.vm], outputs=[self.vm_cost_per_hour, self.tokens_per_second])
self.maxed_out = gr.Slider(minimum=0.01, value=50., step=0.01, label="% maxed out",
info="How much the GPU is fully used.",
interactive=True,
visible=False)
self.used = gr.Slider(minimum=0.01, value=50., step=0.01, label="% used",
info="Percentage of time the GPU is used.",
interactive=True,
visible=False)
def compute_cost_per_token(self, vm_cost_per_hour, tokens_per_second, maxed_out, used):
cost_per_token = vm_cost_per_hour / (tokens_per_second * 3600 * maxed_out * used)
return cost_per_token
class CohereModel(BaseTCOModel):
def __init__(self):
self.set_name("(SaaS) Cohere")
self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$
with:
CT = Cost per output Token
CT_1M = Cost per one million Tokens (from Cohere's pricing web page)
L = Input Length
""")
super().__init__()
def render(self):
def on_use_case_change(use_case):
if use_case == "Summarize":
return gr.Dropdown.update(choices=["Default"])
else:
return gr.Dropdown.update(choices=["Default", "Custom"])
self.use_case = gr.Dropdown(["Embed", "Generate", "Classify", "Summarize"], value="Generate",
label="Use case",
interactive=True, visible=False)
self.model = gr.Dropdown(["Default", "Custom"], value="Default",
label="Model",
interactive=True, visible=False)
self.use_case.change(on_use_case_change, inputs=self.use_case, outputs=self.model)
self.input_length = gr.Number(350, label="Average number of input tokens",
interactive=True, visible=False)
def compute_cost_per_token(self, use_case, model, input_length):
"""Cost per token = """
use_case = use_case[0]
model = model[0]
if use_case == "Embed":
if model == "Default":
cost_per_1M_input_tokens = 0.4
else:
cost_per_1M_input_tokens = 0.8
elif use_case == "Generate":
if model == "Default":
cost_per_1M_input_tokens = 15
else:
cost_per_1M_input_tokens = 30
elif use_case == "Classify":
if model == "Default":
cost_per_1M_input_tokens = 200
else:
cost_per_1M_input_tokens = 200
else:
cost_per_1M_input_tokens = 15
cost_per_output_token = cost_per_1M_input_tokens * input_length / 1000000
return cost_per_output_token
class ModelPage:
def __init__(self, Models: BaseTCOModel):
self.models: list[BaseTCOModel] = []
for Model in Models:
model = Model()
self.models.append(model)
def render(self):
for model in self.models:
model.render()
model.register_components_for_cost_computing()
def get_all_components(self) -> list[Component]:
output = []
for model in self.models:
output += model.get_components()
return output
def get_all_components_for_cost_computing(self) -> list[Component]:
output = []
for model in self.models:
output += model.get_components_for_cost_computing()
return output
def make_model_visible(self, name:str):
# First decide which indexes
output = []
for model in self.models:
if model.get_name() == name:
output+= [gr.update(visible=True)] * len(model.get_components())
else:
output+= [gr.update(visible=False)] * len(model.get_components())
return output
def compute_cost_per_token(self, *args):
begin=0
current_model = args[-1]
for model in self.models:
model_n_args = len(model.get_components_for_cost_computing())
if current_model == model.get_name():
model_args = args[begin:begin+model_n_args]
model_tco = model.compute_cost_per_token(*model_args)
formula = model.get_formula()
return f"Model {current_model} has a TCO of: ${model_tco}", model_tco, formula
begin = begin+model_n_args