import torch from models import VQVAE, build_vae_var import torch.nn as nn import torch.nn.functional as F from transformers import AutoTokenizer, SiglipTextModel from peft import LoraConfig, get_peft_model from torchvision.transforms import ToPILImage import random import gradio as gr class SimpleAdapter(nn.Module): def __init__(self, input_dim=512, hidden_dim=1024, out_dim=1024): super(SimpleAdapter, self).__init__() self.layer1 = nn.Linear(input_dim, hidden_dim) self.norm0 = nn.LayerNorm(input_dim) self.activation1 = nn.GELU() self.layer2 = nn.Linear(hidden_dim, out_dim) self.norm2 = nn.LayerNorm(out_dim) self._initialize_weights() def _initialize_weights(self): for m in self.modules(): if isinstance(m, nn.Linear): nn.init.xavier_uniform_(m.weight, gain=0.001) nn.init.zeros_(m.bias) elif isinstance(m, nn.LayerNorm): nn.init.ones_(m.weight) nn.init.zeros_(m.bias) def forward(self, x): x = self.norm0(x) x = self.layer1(x) x = self.activation1(x) x = self.layer2(x) x = self.norm2(x) return x class InferenceTextVAR(nn.Module): def __init__(self, pl_checkpoint=None, start_class_id=578, hugging_face_token=None, siglip_model='google/siglip-base-patch16-224', device="cpu", MODEL_DEPTH=16): super(InferenceTextVAR, self).__init__() self.device = device self.class_id = start_class_id # Define layers patch_nums = (1, 2, 3, 4, 5, 6, 8, 10, 13, 16) self.vae, self.var = build_vae_var( V=4096, Cvae=32, ch=160, share_quant_resi=4, device=device, patch_nums=patch_nums, num_classes=1000, depth=MODEL_DEPTH, shared_aln=False, ) self.text_processor = AutoTokenizer.from_pretrained(siglip_model, token=hugging_face_token) self.siglip_text_encoder = SiglipTextModel.from_pretrained(siglip_model, token=hugging_face_token).to(device) self.adapter = SimpleAdapter( input_dim=self.siglip_text_encoder.config.hidden_size, out_dim=self.var.C # Ensure dimensional consistency ).to(device) self.apply_lora_to_var() if pl_checkpoint is not None: state_dict = torch.load(pl_checkpoint, map_location="cpu")['state_dict'] var_state_dict = {k[len('var.'):]: v for k, v in state_dict.items() if k.startswith('var.')} vae_state_dict = {k[len('vae.'):]: v for k, v in state_dict.items() if k.startswith('vae.')} adapter_state_dict = {k[len('adapter.'):]: v for k, v in state_dict.items() if k.startswith('adapter.')} self.var.load_state_dict(var_state_dict) self.vae.load_state_dict(vae_state_dict) self.adapter.load_state_dict(adapter_state_dict) del self.vae.encoder def apply_lora_to_var(self): """ Applies LoRA (Low-Rank Adaptation) to the VAR model. """ def find_linear_module_names(model): linear_module_names = [] for name, module in model.named_modules(): if isinstance(module, nn.Linear): linear_module_names.append(name) return linear_module_names linear_module_names = find_linear_module_names(self.var) lora_config = LoraConfig( r=8, lora_alpha=32, target_modules=linear_module_names, lora_dropout=0.05, bias="none", ) self.var = get_peft_model(self.var, lora_config) @torch.no_grad() def generate_image(self, text, beta=1, seed=None, more_smooth=False, top_k=0, top_p=0.5): if seed is None: seed = random.randint(0, 2**32 - 1) inputs = self.text_processor([text], padding="max_length", return_tensors="pt").to(self.device) outputs = self.siglip_text_encoder(**inputs) pooled_output = outputs.pooler_output # pooled (EOS token) states pooled_output = F.normalize(pooled_output, p=2, dim=-1) # Normalize delta condition cond_delta = F.normalize(pooled_output, p=2, dim=-1).to(self.device) # Use correct device cond_delta = self.adapter(cond_delta) cond_delta = F.normalize(cond_delta, p=2, dim=-1) # Normalize delta condition generated_images = self.var.autoregressive_infer_cfg( B=1, label_B=self.class_id, delta_condition=cond_delta[:1], beta=beta, alpha=1, top_k=top_k, top_p=top_p, more_smooth=more_smooth, g_seed=seed ) image = ToPILImage()(generated_images[0].cpu()) return image if __name__ == '__main__': # Initialize the model checkpoint = 'VARtext_v1.pth' # Replace with your actual checkpoint path device = 'cpu' if not torch.cuda.is_available() else 'cuda' model = InferenceTextVAR(device=device) model.load_state_dict(torch.load(checkpoint, map_location=device)) model.to(device) def generate_image_gradio(text, beta=1.0, seed=None, more_smooth=False, top_k=0, top_p=0.9): print(f"Generating image for text: {text}\n" f"beta: {beta}\n" f"seed: {seed}\n" f"more_smooth: {more_smooth}\n" f"top_k: {top_k}\n" f"top_p: {top_p}\n") image = model.generate_image(text, beta=beta, seed=seed, more_smooth=more_smooth, top_k=int(top_k), top_p=top_p) return image with gr.Blocks(css=""" .project-item {margin-bottom: 30px;} .project-description {margin-top: 20px;} .github-button, .huggingface-button, .wandb-button { display: inline-block; margin-left: 10px; text-decoration: none; font-size: 14px; padding: 5px 10px; background-color: #f0f0f0; border-radius: 5px; color: black; } .project-content {display: flex; flex-direction: row;} .project-description {flex: 2; padding-right: 20px;} .project-options-image {flex: 1;} .funko-image {width: 100%; max-width: 300px;} """) as demo: gr.Markdown(""" # PopYou2 - VAR Text [![GitHub](https://img.shields.io/badge/GitHub-Repository-blue?logo=github)](https://github.com/amit154154/VAR_clip) [![Weights & Biases](https://img.shields.io/badge/Weights%20%26%20Biases-Report-orange?logo=weightsandbiases)](https://api.wandb.ai/links/amit154154/cqccmfsl) ## Project Explanation - **Dataset Generation:** Generated a comprehensive dataset of approximately 100,000 Funko Pop! images with detailed prompts using [SDXL Turbo](https://huggingface.co/stabilityai/sdxl-turbo) for high-quality data creation. - **Model Fine-tuning:** Fine-tuned the [Visual AutoRegressive (VAR)](https://arxiv.org/abs/2404.02905) model, pretrained on ImageNet, to adapt it for Funko Pop! generation by injecting a custom embedding representing the "doll" class. - **Adapter Training:** Trained an adapter with the frozen [SigLIP image encoder](https://github.com/FoundationVision/VAR) and a lightweight LoRA module to map image embeddings to text representation in a large language model. - **Text-to-Image Generation:** Enabled text-to-image generation by replacing the SigLIP image encoder with its text encoder, retaining frozen components such as the VAE and generator for efficiency and quality. ## Generate Your Own Funko Pop! """) with gr.Tab("Generate Image"): with gr.Row(): with gr.Column(scale=1): text_input = gr.Textbox(label="Input Text", placeholder="Enter a description for your Funko Pop!") beta_input = gr.Slider(label="Beta", minimum=0.0, maximum=2.5, step=0.05, value=1.0) seed_input = gr.Number(label="Seed", value=None) more_smooth_input = gr.Checkbox(label="More Smooth", value=False) top_k_input = gr.Number(label="Top K", value=0) top_p_input = gr.Slider(label="Top P", minimum=0.0, maximum=1.0, step=0.01, value=0.5) generate_button = gr.Button("Generate Image") with gr.Column(scale=1): image_output = gr.Image(label="Generated Image") generate_button.click( generate_image_gradio, inputs=[text_input, beta_input, seed_input, more_smooth_input, top_k_input, top_p_input], outputs=image_output ) gr.Markdown("## Examples") with gr.Row(): with gr.Column(): gr.Markdown("### Example 1") gr.Markdown("A Funko Pop figure of a yellow robot Tom Cruise with headphones on a white background") example1_image = gr.Image(value="examples/tom_cruise_robot.png") # Replace with the actual path with gr.Column(): gr.Markdown("### Example 2") gr.Markdown("A Funko Pop figure of an alien Scarlett Johansson holding a shield on a white background") example2_image = gr.Image(value="examples/alien_Scarlett_Johansson.png") # Replace with the actual path with gr.Column(): gr.Markdown("### Example 3") gr.Markdown("A Funko Pop figure of a woman with a hat and pink long hair and blue dress on a white background") example3_image = gr.Image(value="examples/woman_pink.png") # Replace with the actual path gr.Markdown(""" ## Customize Your Funko Pop! Build your own Funko Pop! by selecting options below and clicking "Generate Custom Funko Pop!". """) def update_custom_image(famous_name, character, action): # Build the prompt based on the selections parts = [] if famous_name != "None": parts.append(f"a Funko Pop figure of {famous_name}") else: parts.append("a Funko Pop figure") if character != "None": parts.append(f"styled as a {character}") if action != "None": parts.append(f"performing {action}") parts.append("on a white background") prompt = ", ".join(parts) image = model.generate_image(prompt) return image famous_name_input = gr.Dropdown(choices=["None", "Donald Trump", "Johnny Depp", "Oprah Winfrey,Lebron James"], label="Famous Name", value="None") character_input = gr.Dropdown(choices=["None", "Alien", "Robot"], label="Character", value="None") action_input = gr.Dropdown(choices=["None", "Playing the Guitar", "Holding the Sword","wearing headphone"], label="Action", value="None") custom_generate_button = gr.Button("Generate Custom Funko Pop!") custom_image_output = gr.Image(label="Custom Funko Pop!") custom_generate_button.click( update_custom_image, inputs=[famous_name_input, character_input, action_input], outputs=custom_image_output ) demo.launch()