|
--- |
|
license: gpl-3.0 |
|
--- |
|
|
|
# Easy CLIP Embeddings alignment with Guannaco Models |
|
|
|
```python |
|
pipe = StableDiffusionPipeline(...) |
|
# llm_model = load_quant('/GuanacoOnConsumerHardware', 'guanaco7b-4bit-128g.pt', 4, 128, 0) |
|
llm_tokenizer = LlamaTokenizer.from_pretrained("JosephusCheung/Guanaco",use_fast=False,torch_dtype=torch.float16) |
|
llm_model = LlamaForCausalLM.from_pretrained("JosephusCheung/Guanaco",device_map="auto",torch_dtype=torch.float16) |
|
|
|
class LLMToCLIP(nn.Module): |
|
def __init__(self): |
|
super(LLMToCLIP, self).__init__() |
|
self.proj = nn.Linear(4096, 4096, bias=False) |
|
self.deproj = nn.Linear(4096, 768, bias=False) |
|
|
|
def forward(self, x): |
|
a = self.proj(x) |
|
b = self.deproj(a) |
|
return b |
|
|
|
llm_to_clip=LLMToCLIP() |
|
llm_to_clip.load_state_dict(torch.load("toclip.pth")) |
|
|
|
llm_embeddings = llm_model(input_ids=input_ids, output_hidden_states=True).hidden_states[-1] |
|
|
|
image = pipe(prompt_embeds=llm_to_clip(llm_embeddings)).images[0] |
|
``` |
|
|
