|
from gradio.components import Component |
|
import gradio as gr |
|
from abc import ABC, abstractclassmethod |
|
import inspect |
|
|
|
class BaseTCOModel(ABC): |
|
|
|
def __setattr__(self, name, value): |
|
if isinstance(value, Component): |
|
self._components.append(value) |
|
self.__dict__[name] = value |
|
|
|
def __init__(self): |
|
super(BaseTCOModel, self).__setattr__("_components", []) |
|
|
|
def get_components(self) -> list[Component]: |
|
return self._components |
|
|
|
def get_components_for_cost_computing(self): |
|
return self.components_for_cost_computing |
|
|
|
def get_name(self): |
|
return self.name |
|
|
|
def register_components_for_cost_computing(self): |
|
args = inspect.getfullargspec(self.compute_cost_per_token)[0][1:] |
|
self.components_for_cost_computing = [self.__getattribute__(arg) for arg in args] |
|
|
|
@abstractclassmethod |
|
def compute_cost_per_token(self): |
|
pass |
|
|
|
@abstractclassmethod |
|
def render(self): |
|
pass |
|
|
|
def set_name(self, name): |
|
self.name = name |
|
|
|
def set_formula(self, formula): |
|
self.formula = formula |
|
|
|
def get_formula(self): |
|
return self.formula |
|
|
|
class OpenAIModel(BaseTCOModel): |
|
|
|
def __init__(self): |
|
self.set_name("(SaaS) OpenAI") |
|
self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$ <br> |
|
with: <br> |
|
CT = Cost per output Token <br> |
|
CT_1K = Cost per 1000 Tokens (from OpenAI's pricing web page) <br> |
|
L = Input Length |
|
""") |
|
super().__init__() |
|
|
|
def render(self): |
|
def on_model_change(model): |
|
|
|
if model == "GPT-4": |
|
return gr.Dropdown.update(choices=["8K", "32K"]) |
|
else: |
|
return gr.Dropdown.update(choices=["4K", "16K"]) |
|
|
|
self.model = gr.Dropdown(["GPT-4", "GPT-3.5 Turbo"], value="GPT-4", |
|
label="OpenAI models", |
|
interactive=True, visible=False) |
|
self.context_length = gr.Dropdown(["8K", "32K"], value="8K", interactive=True, |
|
label="Context size", |
|
visible=False, info="Number of tokens the model considers when processing text") |
|
self.model.change(on_model_change, inputs=self.model, outputs=self.context_length) |
|
self.input_length = gr.Number(350, label="Average number of input tokens", |
|
interactive=True, visible=False) |
|
|
|
def compute_cost_per_token(self, model, context_length, input_length): |
|
"""Cost per token = """ |
|
model = model[0] |
|
context_length = context_length[0] |
|
|
|
if model == "GPT-4" and context_length == "8K": |
|
cost_per_1k_input_tokens = 0.03 |
|
elif model == "GPT-4" and context_length == "32K": |
|
cost_per_1k_input_tokens = 0.06 |
|
elif model == "GPT-3.5" and context_length == "4K": |
|
cost_per_1k_input_tokens = 0.0015 |
|
else: |
|
cost_per_1k_input_tokens = 0.003 |
|
|
|
cost_per_output_token = cost_per_1k_input_tokens * input_length / 1000 |
|
|
|
return cost_per_output_token |
|
|
|
class OpenSourceLlama2Model(BaseTCOModel): |
|
|
|
def __init__(self): |
|
self.set_name("(Open source) Llama 2") |
|
self.set_formula(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br> |
|
with: <br> |
|
CT = Cost per Token <br> |
|
VM_CH = VM Cost per Hour <br> |
|
TS = Tokens per Second <br> |
|
MO = Maxed Out <br> |
|
U = Used |
|
""") |
|
super().__init__() |
|
|
|
def render(self): |
|
vm_choices = ["1x Nvidia A100 (Azure NC24ads A100 v4)", |
|
"2x Nvidia A100 (Azure NC48ads A100 v4)", |
|
"4x Nvidia A100 (Azure NC48ads A100 v4)"] |
|
|
|
def on_model_change(model): |
|
if model == "Llama 2 7B": |
|
return [gr.Dropdown.update(choices=vm_choices), |
|
gr.Markdown.update(value="To see the script used to benchmark the Llama2-7B model, [click here](https://example.com/script)"), |
|
gr.Number.update(value=3.6730), |
|
gr.Number.update(value=694.38), |
|
gr.Number.update(visible=True) |
|
] |
|
else: |
|
not_supported_vm = ["1x Nvidia A100 (Azure NC24ads A100 v4)", "2x Nvidia A100 (Azure NC48ads A100 v4)"] |
|
choices = [x for x in vm_choices if x not in not_supported_vm] |
|
return [gr.Dropdown.update(choices=choices, value="4x Nvidia A100 (Azure NC48ads A100 v4)"), |
|
gr.Markdown.update(value="To see the benchmark results used for the Llama2-70B model, [click here](https://www.cursor.so/blog/llama-inference#user-content-fn-llama-paper)"), |
|
gr.Number.update(value=14.692), |
|
gr.Number.update(value=18.6), |
|
gr.Number.update(visible=False) |
|
] |
|
|
|
def on_vm_change(model, vm): |
|
|
|
if model == "Llama 2 7B" and vm == "1x Nvidia A100 (Azure NC24ads A100 v4)": |
|
return [gr.Number.update(value=3.6730), gr.Number.update(value=694.38)] |
|
elif model == "Llama 2 7B" and vm == "2x Nvidia A100 (Azure NC48ads A100 v4)": |
|
return [gr.Number.update(value=7.346), gr.Number.update(value=1388.76)] |
|
elif model == "Llama 2 7B" and vm == "4x Nvidia A100 (Azure NC48ads A100 v4)": |
|
return [gr.Number.update(value=14.692), gr.Number.update(value=2777.52)] |
|
elif model == "Llama 2 70B" and vm == "4x Nvidia A100 (Azure NC48ads A100 v4)": |
|
return [gr.Number.update(value=14.692), gr.Number.update(value=18.6)] |
|
|
|
self.model = gr.Dropdown(["Llama 2 7B", "Llama 2 70B"], value="Llama 2 7B", label="OpenSource models", visible=False) |
|
self.vm = gr.Dropdown(vm_choices, |
|
value="1x Nvidia A100 (Azure NC24ads A100 v4)", |
|
visible=False, |
|
label="Instance of VM with GPU", |
|
info="Your options for this choice depend on the model you previously chose" |
|
) |
|
self.vm_cost_per_hour = gr.Number(3.6730, label="VM instance cost per hour", |
|
interactive=False, visible=False) |
|
self.tokens_per_second = gr.Number(694.38, visible=False, |
|
label="Number of tokens per second for this specific model and VM instance", |
|
interactive=False |
|
) |
|
self.input_length = gr.Number(233, label="Average number of input tokens", info="This is the number of input tokens used when the model was benchmarked to get the number of tokens/second it processes", |
|
interactive=False, visible=False) |
|
self.info = gr.Markdown("To see the script used to benchmark the Llama2-7B model, [click here](https://example.com/script)", interactive=False, visible=False) |
|
|
|
self.model.change(on_model_change, inputs=self.model, outputs=[self.vm, self.info, self.vm_cost_per_hour, self.tokens_per_second, self.input_length]) |
|
self.vm.change(on_vm_change, inputs=[self.model, self.vm], outputs=[self.vm_cost_per_hour, self.tokens_per_second]) |
|
self.maxed_out = gr.Slider(minimum=0.01, value=50., step=0.01, label="% maxed out", |
|
info="How much the GPU is fully used", |
|
interactive=True, |
|
visible=False) |
|
self.used = gr.Slider(minimum=0.01, value=50., step=0.01, label="% used", |
|
info="Percentage of time the GPU is used", |
|
interactive=True, |
|
visible=False) |
|
|
|
def compute_cost_per_token(self, vm_cost_per_hour, tokens_per_second, maxed_out, used): |
|
cost_per_token = vm_cost_per_hour / (tokens_per_second * 3600 * maxed_out * used) |
|
return cost_per_token |
|
|
|
class OpenSourceDIY(BaseTCOModel): |
|
|
|
def __init__(self): |
|
self.set_name("(Open source) DIY") |
|
self.set_formula(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br> |
|
with: <br> |
|
CT = Cost per Token <br> |
|
VM_CH = VM Cost per Hour <br> |
|
TS = Tokens per Second <br> |
|
MO = Maxed Out <br> |
|
U = Used |
|
""") |
|
super().__init__() |
|
|
|
def render(self): |
|
self.info = gr.Markdown("Compute the cost/token based on our formula below, using your own parameters", visible=False) |
|
self.display_formula = gr.Markdown(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br> |
|
with: <br> |
|
CT = Cost per Token <br> |
|
VM_CH = VM Cost per Hour <br> |
|
TS = Tokens per Second <br> |
|
MO = Maxed Out <br> |
|
U = Used |
|
""", visible=False) |
|
self.vm_cost_per_hour = gr.Number(3.5, label="VM instance cost per hour", |
|
interactive=True, visible=False) |
|
self.tokens_per_second = gr.Number(700, visible=False, |
|
label="Number of tokens per second for this specific model and VM instance", |
|
interactive=True |
|
) |
|
self.maxed_out = gr.Slider(minimum=0.01, value=50., step=0.01, label="% maxed out", |
|
info="How much the GPU is fully used", |
|
interactive=True, |
|
visible=False) |
|
self.used = gr.Slider(minimum=0.01, value=50., step=0.01, label="% used", |
|
info="Percentage of time the GPU is used", |
|
interactive=True, |
|
visible=False) |
|
|
|
def compute_cost_per_token(self, vm_cost_per_hour, tokens_per_second, maxed_out, used): |
|
cost_per_token = vm_cost_per_hour / (tokens_per_second * 3600 * maxed_out * used) |
|
return cost_per_token |
|
|
|
class CohereModel(BaseTCOModel): |
|
|
|
def __init__(self): |
|
self.set_name("(SaaS) Cohere") |
|
self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$ <br> |
|
with: <br> |
|
CT = Cost per output Token <br> |
|
CT_1M = Cost per one million Tokens (from Cohere's pricing web page) <br> |
|
L = Input Length |
|
""") |
|
super().__init__() |
|
|
|
def render(self): |
|
def on_use_case_change(use_case): |
|
if use_case == "Summarize": |
|
return gr.Dropdown.update(choices=["Default"]) |
|
else: |
|
return gr.Dropdown.update(choices=["Default", "Custom"]) |
|
|
|
self.use_case = gr.Dropdown(["Generate", "Summarize"], value="Generate", |
|
label="API", |
|
interactive=True, visible=False) |
|
self.model = gr.Dropdown(["Default", "Custom"], value="Default", |
|
label="Model", |
|
interactive=True, visible=False) |
|
self.use_case.change(on_use_case_change, inputs=self.use_case, outputs=self.model) |
|
self.input_length = gr.Number(350, label="Average number of input tokens", |
|
interactive=True, visible=False) |
|
|
|
def compute_cost_per_token(self, use_case, model, input_length): |
|
"""Cost per token = """ |
|
use_case = use_case[0] |
|
model = model[0] |
|
|
|
if use_case == "Generate": |
|
if model == "Default": |
|
cost_per_1M_input_tokens = 15 |
|
else: |
|
cost_per_1M_input_tokens = 30 |
|
else: |
|
cost_per_1M_input_tokens = 15 |
|
|
|
cost_per_output_token = cost_per_1M_input_tokens * input_length / 1000000 |
|
|
|
return cost_per_output_token |
|
|
|
class ModelPage: |
|
|
|
def __init__(self, Models: BaseTCOModel): |
|
self.models: list[BaseTCOModel] = [] |
|
for Model in Models: |
|
model = Model() |
|
self.models.append(model) |
|
|
|
def render(self): |
|
for model in self.models: |
|
model.render() |
|
model.register_components_for_cost_computing() |
|
|
|
def get_all_components(self) -> list[Component]: |
|
output = [] |
|
for model in self.models: |
|
output += model.get_components() |
|
return output |
|
|
|
def get_all_components_for_cost_computing(self) -> list[Component]: |
|
output = [] |
|
for model in self.models: |
|
output += model.get_components_for_cost_computing() |
|
return output |
|
|
|
def make_model_visible(self, name:str): |
|
|
|
output = [] |
|
for model in self.models: |
|
if model.get_name() == name: |
|
output+= [gr.update(visible=True)] * len(model.get_components()) |
|
else: |
|
output+= [gr.update(visible=False)] * len(model.get_components()) |
|
return output |
|
|
|
def compute_cost_per_token(self, *args): |
|
begin=0 |
|
current_model = args[-1] |
|
for model in self.models: |
|
model_n_args = len(model.get_components_for_cost_computing()) |
|
if current_model == model.get_name(): |
|
|
|
model_args = args[begin:begin+model_n_args] |
|
model_tco = model.compute_cost_per_token(*model_args) |
|
formula = model.get_formula() |
|
return f"Model {current_model} has a TCO of: ${model_tco}", model_tco, formula |
|
|
|
begin = begin+model_n_args |