import sys import gradio as gr import jax from huggingface_hub import snapshot_download from PIL import Image from transformers import AutoTokenizer import torch from torchvision.io import ImageReadMode, read_image LOCAL_PATH = snapshot_download("flax-community/medclip") sys.path.append(LOCAL_PATH) from src.modeling_medclip import FlaxMedCLIP from run_medclip import Transform def prepare_image(image_path, model): image = read_image(image_path, mode=ImageReadMode.RGB) preprocess = Transform(model.config.vision_config.image_size) preprocess = torch.jit.script(preprocess) preprocessed_image = preprocess(image) pixel_values = torch.stack([preprocessed_image]).permute(0, 2, 3, 1).numpy() return pixel_values def prepare_text(text, tokenizer): return tokenizer(text, return_tensors="np") def save_file_to_disk(uplaoded_file): temp_file = "/tmp/image.jpeg" im = Image.fromarray(uplaoded_file) im.save(temp_file) return temp_file def load_tokenizer_and_model(): # load the saved model tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased") model = FlaxMedCLIP.from_pretrained(LOCAL_PATH) return tokenizer, model def run_inference(image_path, text, model, tokenizer): pixel_values = prepare_image(image_path, model) input_text = prepare_text(text, tokenizer) model_output = model( input_text["input_ids"], pixel_values, attention_mask=input_text["attention_mask"], train=False, return_dict=True, ) logits = model_output["logits_per_image"] score = jax.nn.sigmoid(logits)[0][0] return score tokenizer, model = load_tokenizer_and_model() def score_image_caption_pair(uploaded_file, text_input): local_image_path = save_file_to_disk(uploaded_file) score = run_inference( local_image_path, text_input, model, tokenizer).tolist() return {"Score": score} image = gr.inputs.Image(shape=(299, 299)) iface = gr.Interface( fn=score_image_caption_pair, inputs=[image, "text"], outputs=["label"], allow_flagging=False, title="Medical diagnosis evaluation via MedCLIP", description=""" The purpose of this demo is to help medical students measure their diagnostic capabilities in purely academic settings. Under no circumstances should it be used to make a self-diagnosis or confront a real doctor. """ ) iface.launch()