app.py

import gradio as gr
from transformers import AutoProcessor, Pix2StructForConditionalGeneration, T5Tokenizer, T5ForConditionalGeneration, Pix2StructProcessor, BartConfig,ViTConfig,VisionEncoderDecoderConfig, DonutProcessor, VisionEncoderDecoderModel, AutoTokenizer, AutoModel
from PIL import Image  
import torch
import warnings
import re
import json
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import argparse
from scipy import optimize
from typing import Optional
import dataclasses
import editdistance
import itertools
import sys
import time
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger()

warnings.filterwarnings('ignore')
MAX_PATCHES = 512
# Load the models and processor
#device = torch.device("cpu")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Paths to the models
ko_deplot_model_path = './deplot_model_ver_kor_24.7.25_refinetuning_epoch1.bin'
aihub_deplot_model_path='./deplot_k.pt'
t5_model_path = './ke_t5.pt'

# Load first model ko-deplot
processor1 = Pix2StructProcessor.from_pretrained('nuua/ko-deplot')
model1 = Pix2StructForConditionalGeneration.from_pretrained('nuua/ko-deplot')
model1.load_state_dict(torch.load(ko_deplot_model_path, map_location=device))
model1.to(device)

# Load second model aihub-deplot
processor2 = AutoProcessor.from_pretrained("ybelkada/pix2struct-base")
model2 = Pix2StructForConditionalGeneration.from_pretrained("ybelkada/pix2struct-base")
model2.load_state_dict(torch.load(aihub_deplot_model_path, map_location=device))


tokenizer = T5Tokenizer.from_pretrained("KETI-AIR/ke-t5-base")
t5_model = T5ForConditionalGeneration.from_pretrained("KETI-AIR/ke-t5-base")
t5_model.load_state_dict(torch.load(t5_model_path, map_location=device))

model2.to(device)
t5_model.to(device)

#Load third model unichart
unichart_model_path = "./unichart"
model3 =  VisionEncoderDecoderModel.from_pretrained(unichart_model_path)
processor3 = DonutProcessor.from_pretrained(unichart_model_path)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model3.to(device)  

#ko-deplot 추론함수
# Function to format output
def format_output(prediction):
    return prediction.replace('<0x0A>', '\n')

# First model prediction ko-deplot
def predict_model1(image):
    images = [image]
    inputs = processor1(images=images, text="What is the title of the chart", return_tensors="pt", padding=True)
    inputs = {k: v.to(device) for k, v in inputs.items()}  # Move to GPU

    model1.eval()
    with torch.no_grad():
        predictions = model1.generate(**inputs, max_new_tokens=4096)
    outputs = [processor1.decode(pred, skip_special_tokens=True) for pred in predictions]

    formatted_output = format_output(outputs[0])
    return formatted_output


def replace_unk(text):
    # 1. '제목:', '유형:' 글자 앞에 있는 <unk>는 \n로 바꿈
    text = re.sub(r'<unk>(?=제목:|유형:)', '\n', text)
    # 2. '세로 ' 또는 '가로 '와 '대형' 사이에 있는 <unk>를 ""로 바꿈
    text = re.sub(r'(?<=세로 |가로 )<unk>(?=대형)', '', text)
    # 3. 숫자와 텍스트 사이에 있는 <unk>를 \n로 바꿈
    text = re.sub(r'(\d)<unk>([^\d])', r'\1\n\2', text)
    # 4. %, 원, 건, 명 뒤에 나오는 <unk>를 \n로 바꿈
    text = re.sub(r'(?<=[%원건명\)])<unk>', '\n', text)
    # 5. 숫자와 숫자 사이에 있는 <unk>를 \n로 바꿈
    text = re.sub(r'(\d)<unk>(\d)', r'\1\n\2', text)
    # 6. '형'이라는 글자와 ' |' 사이에 있는 <unk>를 \n로 바꿈
    text = re.sub(r'형<unk>(?= \|)', '형\n', text)
    # 7. 나머지 <unk>를 모두 ""로 바꿈
    text = text.replace('<unk>', '')
    return text

# Second model prediction aihub_deplot
def predict_model2(image):
    image = image.convert("RGB")
    inputs = processor2(images=image, return_tensors="pt", max_patches=MAX_PATCHES).to(device)

    flattened_patches = inputs.flattened_patches.to(device)
    attention_mask = inputs.attention_mask.to(device)

    model2.eval()
    t5_model.eval()
    with torch.no_grad():
        deplot_generated_ids = model2.generate(flattened_patches=flattened_patches, attention_mask=attention_mask, max_length=1000)
    generated_datatable = processor2.batch_decode(deplot_generated_ids, skip_special_tokens=False)[0]
    generated_datatable = generated_datatable.replace("<pad>", "<unk>").replace("</s>", "<unk>")
    refined_table = replace_unk(generated_datatable)
    return refined_table

def predict_model3(image):
    image=image.convert("RGB")
    input_prompt = "<extract_data_table> <s_answer>"
    decoder_input_ids = processor3.tokenizer(input_prompt, add_special_tokens=False, return_tensors="pt").input_ids
    pixel_values = processor3(image, return_tensors="pt").pixel_values
    outputs = model3.generate(
    pixel_values.to(device),
    decoder_input_ids=decoder_input_ids.to(device),
    max_length=model3.decoder.config.max_position_embeddings,
    early_stopping=True,
    pad_token_id=processor3.tokenizer.pad_token_id,
    eos_token_id=processor3.tokenizer.eos_token_id,
    use_cache=True,
    num_beams=4,
    bad_words_ids=[[processor3.tokenizer.unk_token_id]],
    return_dict_in_generate=True,
    )
    sequence = processor3.batch_decode(outputs.sequences)[0]
    sequence = sequence.replace(processor3.tokenizer.eos_token, "").replace(processor3.tokenizer.pad_token, "")
    sequence = sequence.split("<s_answer>")[-1].strip()

    return sequence
#function for converting aihub dataset labeling json file to ko-deplot data table
def process_json_file(input_file):
    with open(input_file, 'r', encoding='utf-8') as file:
        data = json.load(file)

    # 필요한 데이터 추출
    chart_type = data['metadata']['chart_sub']
    title = data['annotations'][0]['title']
    x_axis = data['annotations'][0]['axis_label']['x_axis']
    y_axis = data['annotations'][0]['axis_label']['y_axis']
    legend = data['annotations'][0]['legend']
    data_labels = data['annotations'][0]['data_label']
    is_legend = data['annotations'][0]['is_legend']

    # 원하는 형식으로 변환
    formatted_string = f"TITLE | {title} <0x0A> "
    if '가로' in chart_type:
        if is_legend:
            # 가로 차트 처리
            formatted_string += " | ".join(legend) + " <0x0A> "
            for i in range(len(y_axis)):
                row = [y_axis[i]]
                for j in range(len(legend)):
                    if i < len(data_labels[j]):
                        row.append(str(data_labels[j][i]))  # 데이터 값을 문자열로 변환
                    else:
                        row.append("")  # 데이터가 없는 경우 빈 문자열 추가
                formatted_string += " | ".join(row) + " <0x0A> "
        else:
            # is_legend가 False인 경우
            for i in range(len(y_axis)):
                row = [y_axis[i], str(data_labels[0][i])]
                formatted_string += " | ".join(row) + " <0x0A> "
    elif chart_type == "원형":
        # 원형 차트 처리
        if legend:
            used_labels = legend
        else:
            used_labels = x_axis

        formatted_string += " | ".join(used_labels) + " <0x0A> "
        row = [data_labels[0][i] for i in range(len(used_labels))]
        formatted_string += " | ".join(row) + " <0x0A> "
    elif chart_type == "혼합형":
        # 혼합형 차트 처리
        all_legends = [ann['legend'][0] for ann in data['annotations']]
        formatted_string += " | ".join(all_legends) + " <0x0A> "

        combined_data = []
        for i in range(len(x_axis)):
            row = [x_axis[i]]
            for ann in data['annotations']:
                if i < len(ann['data_label'][0]):
                    row.append(str(ann['data_label'][0][i]))  # 데이터 값을 문자열로 변환
                else:
                    row.append("")  # 데이터가 없는 경우 빈 문자열 추가
            combined_data.append(" | ".join(row))

        formatted_string += " <0x0A> ".join(combined_data) + " <0x0A> "
    else:
        # 기타 차트 처리
        if is_legend:
            formatted_string += " | ".join(legend) + " <0x0A> "
            for i in range(len(x_axis)):
                row = [x_axis[i]]
                for j in range(len(legend)):
                    if i < len(data_labels[j]):
                        row.append(str(data_labels[j][i]))  # 데이터 값을 문자열로 변환
                    else:
                        row.append("")  # 데이터가 없는 경우 빈 문자열 추가
                formatted_string += " | ".join(row) + " <0x0A> "
        else:
            for i in range(len(x_axis)):
                if i < len(data_labels[0]):
                    formatted_string += f"{x_axis[i]} | {str(data_labels[0][i])} <0x0A> "
                else:
                    formatted_string += f"{x_axis[i]} |  <0x0A> "  # 데이터가 없는 경우 빈 문자열 추가

    # 마지막 "<0x0A> " 제거
    formatted_string = formatted_string[:-8]
    return format_output(formatted_string)

def chart_data(data):
    datatable = []
    num = len(data)
    for n in range(num):
        title = data[n]['title'] if data[n]['is_title'] else ''
        legend = data[n]['legend'] if data[n]['is_legend'] else ''
        datalabel = data[n]['data_label'] if data[n]['is_datalabel'] else [0]
        unit = data[n]['unit'] if data[n]['is_unit'] else ''
        base = data[n]['base'] if data[n]['is_base'] else ''
        x_axis_title = data[n]['axis_title']['x_axis']
        y_axis_title = data[n]['axis_title']['y_axis']
        x_axis = data[n]['axis_label']['x_axis'] if data[n]['is_axis_label_x_axis'] else [0]
        y_axis = data[n]['axis_label']['y_axis'] if data[n]['is_axis_label_y_axis'] else [0]

        if len(legend) > 1:
            datalabel = np.array(datalabel).transpose().tolist()

        datatable.append([title, legend, datalabel, unit, base, x_axis_title, y_axis_title, x_axis, y_axis])

    return datatable

def datatable(data, chart_type):
    data_table = ''
    num = len(data)

    if len(data)  == 2:
        temp = []
        temp.append(f"대상: {data[0][4]}")
        temp.append(f"제목: {data[0][0]}")
        temp.append(f"유형: {' '.join(chart_type[0:2])}")
        temp.append(f"{data[0][5]} | {data[0][1][0]}({data[0][3]}) | {data[1][1][0]}({data[1][3]})")

        x_axis = data[0][7]
        for idx, x in enumerate(x_axis):
            temp.append(f"{x} | {data[0][2][0][idx]} | {data[1][2][0][idx]}")

        data_table = '\n'.join(temp)
    else:
        for n in range(num):
            temp = []

            title, legend, datalabel, unit, base, x_axis_title, y_axis_title, x_axis, y_axis = data[n]
            legend = [element + f"({unit})" for element in legend]

            if len(legend) > 1:
                temp.append(f"대상: {base}")
                temp.append(f"제목: {title}")
                temp.append(f"유형: {' '.join(chart_type[0:2])}")
                temp.append(f"{x_axis_title} | {' | '.join(legend)}")

                if chart_type[2] == "원형":
                    datalabel = sum(datalabel, [])
                    temp.append(f"{' | '.join([str(d) for d in datalabel])}")
                    data_table = '\n'.join(temp)
                else:
                    axis = y_axis if chart_type[2] == "가로 막대형" else x_axis
                    for idx, (x, d) in enumerate(zip(axis, datalabel)):
                        temp_d = [str(e) for e in d]
                        temp_d = " | ".join(temp_d)
                        row = f"{x} | {temp_d}"
                        temp.append(row)
                    data_table = '\n'.join(temp)
            else:
                temp.append(f"대상: {base}")
                temp.append(f"제목: {title}")
                temp.append(f"유형: {' '.join(chart_type[0:2])}")
                temp.append(f"{x_axis_title} | {unit}")
                axis = y_axis if chart_type[2] == "가로 막대형" else x_axis
                datalabel = datalabel[0]

                for idx, x in enumerate(axis):
                    row = f"{x} | {str(datalabel[idx])}"
                    temp.append(row)
                data_table = '\n'.join(temp)

    return data_table

#function for converting aihub dataset labeling json file to aihub-deplot data table
def process_json_file2(input_file):
    with open(input_file, 'r', encoding='utf-8') as file:
        data = json.load(file)
    # 필요한 데이터 추출
    chart_multi = data['metadata']['chart_multi']
    chart_main = data['metadata']['chart_main']
    chart_sub = data['metadata']['chart_sub']
    chart_type = [chart_multi, chart_sub, chart_main]
    chart_annotations = data['annotations']

    charData = chart_data(chart_annotations)
    dataTable = datatable(charData, chart_type)
    return dataTable

# RMS
def _to_float(text):  # 단위 떼고 숫자만..?
  try:
    if text.endswith("%"):
      # Convert percentages to floats.
      return float(text.rstrip("%")) / 100.0
    else:
      return float(text)
  except ValueError:
    return None


def _get_relative_distance(
    target, prediction, theta = 1.0
):
  """Returns min(1, |target-prediction|/|target|)."""
  if not target:
    return int(not prediction)
  distance = min(abs((target - prediction) / target), 1)
  return distance if distance < theta else 1

def anls_metric(target: str, prediction: str, theta: float = 0.5):
    edit_distance = editdistance.eval(target, prediction)
    normalize_ld = edit_distance / max(len(target), len(prediction))
    return 1 - normalize_ld if normalize_ld < theta else 0

def _permute(values, indexes):
    return tuple(values[i] if i < len(values) else "" for i in indexes)


@dataclasses.dataclass(frozen=True)
class Table:
  """Helper class for the content of a markdown table."""

  base: Optional[str] = None
  title: Optional[str] = None
  chartType: Optional[str] = None
  headers: tuple[str, Ellipsis] = dataclasses.field(default_factory=tuple)
  rows: tuple[tuple[str, Ellipsis], Ellipsis] = dataclasses.field(default_factory=tuple)

  def permuted(self, indexes):
    """Builds a version of the table changing the column order."""
    return Table(
        base=self.base,
        title=self.title,
        chartType=self.chartType,
        headers=_permute(self.headers, indexes),
        rows=tuple(_permute(row, indexes) for row in self.rows),
    )

  def aligned(
      self, headers, text_theta = 0.5
  ):
    """Builds a column permutation with headers in the most correct order."""
    if len(headers) != len(self.headers):
      raise ValueError(f"Header length {headers} must match {self.headers}.")
    distance = []
    for h2 in self.headers:
      distance.append(
          [
              1 - anls_metric(h1, h2, text_theta)
              for h1 in headers
          ]
      )
    cost_matrix = np.array(distance)
    row_ind, col_ind = optimize.linear_sum_assignment(cost_matrix)
    permutation = [idx for _, idx in sorted(zip(col_ind, row_ind))]
    score = (1 - cost_matrix)[permutation[1:], range(1, len(row_ind))].prod()
    return self.permuted(permutation), score

def _parse_table(text, transposed = False): # 표 제목, 열 이름, 행 찾기
  """Builds a table from a markdown representation."""
  lines = text.lower().splitlines()
  if not lines:
    return Table()

  if lines[0].startswith("대상: "):
      base = lines[0][len("대상: ") :].strip()
      offset = 1 #
  else:
    base = None
    offset = 0
  if lines[1].startswith("제목: "):
    title = lines[1][len("제목: ") :].strip()
    offset = 2 #
  else:
    title = None
    offset = 1
  if lines[2].startswith("유형: "):
    chartType = lines[2][len("유형: ") :].strip()
    offset = 3 #
  else:
    chartType = None

  if len(lines) < offset + 1:
    return Table(base=base, title=title, chartType=chartType)

  rows = []
  for line in lines[offset:]:
    rows.append(tuple(v.strip() for v in line.split(" | ")))
  if transposed:
    rows = [tuple(row) for row in itertools.zip_longest(*rows, fillvalue="")]
  return Table(base=base, title=title, chartType=chartType, headers=rows[0], rows=tuple(rows[1:]))

def _get_table_datapoints(table):
    datapoints = {}
    if table.base is not None:
        datapoints["대상"] = table.base
    if table.title is not None:
      datapoints["제목"] = table.title
    if table.chartType is not None:
      datapoints["유형"] = table.chartType
    if not table.rows or len(table.headers) <= 1:
        return datapoints
    for row in table.rows:
        for header, cell in zip(table.headers[1:], row[1:]):
            #print(f"{row[0]} {header} >> {cell}")
            datapoints[f"{row[0]} {header}"] = cell #
    return datapoints

def _get_datapoint_metric(  #
    target,
    prediction,
    text_theta=0.5,
    number_theta=0.1,
):
  """Computes a metric that scores how similar two datapoint pairs are."""
  key_metric = anls_metric(
      target[0], prediction[0], text_theta
  )
  pred_float = _to_float(prediction[1]) # 숫자인지 확인
  target_float = _to_float(target[1])
  if pred_float is not None and target_float:
    return key_metric * (
        1 - _get_relative_distance(target_float, pred_float, number_theta)  # 숫자면 상대적 거리값 계산
    )
  elif target[1] == prediction[1]:
    return key_metric
  else:
    return key_metric * anls_metric(
        target[1], prediction[1], text_theta
    )

def _table_datapoints_precision_recall_f1(  # 찐 계산
    target_table,
    prediction_table,
    text_theta = 0.5,
    number_theta = 0.1,
):
  """Calculates matching similarity between two tables as dicts."""
  target_datapoints = list(_get_table_datapoints(target_table).items())
  prediction_datapoints = list(_get_table_datapoints(prediction_table).items())
  if not target_datapoints and not prediction_datapoints:
    return 1, 1, 1
  if not target_datapoints:
    return 0, 1, 0
  if not prediction_datapoints:
    return 1, 0, 0
  distance = []
  for t, _ in target_datapoints:
    distance.append(
        [
            1 - anls_metric(t, p, text_theta)
            for p, _ in prediction_datapoints
        ]
    )
  cost_matrix = np.array(distance)
  row_ind, col_ind = optimize.linear_sum_assignment(cost_matrix)
  score = 0
  for r, c in zip(row_ind, col_ind):
    score += _get_datapoint_metric(
        target_datapoints[r], prediction_datapoints[c], text_theta, number_theta
    )
  if score == 0:
    return 0, 0, 0
  precision = score / len(prediction_datapoints)
  recall = score / len(target_datapoints)
  return precision, recall, 2 * precision * recall / (precision + recall)

def table_datapoints_precision_recall_per_point(  # 각각 계산...
    targets,
    predictions,
    text_theta = 0.5,
    number_theta = 0.1,
):
  """Computes precisin recall and F1 metrics given two flattened tables.

  Parses each string into a dictionary of keys and values using row and column
  headers. Then we match keys between the two dicts as long as their relative
  levenshtein distance is below a threshold. Values are also compared with
  ANLS if strings or relative distance if they are numeric.

  Args:
    targets: list of list of strings.
    predictions: list of strings.
    text_theta: relative edit distance above this is set to the maximum of 1.
    number_theta: relative error rate above this is set to the maximum of 1.

  Returns:
    Dictionary with per-point precision, recall and F1
  """
  assert len(targets) == len(predictions)
  per_point_scores = {"precision": [], "recall": [], "f1": []}
  for pred, target in zip(predictions, targets):
    all_metrics = []
    for transposed in [True, False]:
      pred_table = _parse_table(pred, transposed=transposed)
      target_table = _parse_table(target, transposed=transposed)

      all_metrics.extend([_table_datapoints_precision_recall_f1(target_table, pred_table, text_theta, number_theta)])

    p, r, f = max(all_metrics, key=lambda x: x[-1])
    per_point_scores["precision"].append(p)
    per_point_scores["recall"].append(r)
    per_point_scores["f1"].append(f)
  return per_point_scores

def table_datapoints_precision_recall(  # deplot 성능지표
    targets,
    predictions,
    text_theta = 0.5,
    number_theta = 0.1,
):
  """Aggregated version of table_datapoints_precision_recall_per_point().

  Same as table_datapoints_precision_recall_per_point() but returning aggregated
  scores instead of per-point scores.

  Args:
    targets: list of list of strings.
    predictions: list of strings.
    text_theta: relative edit distance above this is set to the maximum of 1.
    number_theta: relative error rate above this is set to the maximum of 1.

  Returns:
    Dictionary with aggregated precision, recall and F1
  """
  score_dict = table_datapoints_precision_recall_per_point(
      targets, predictions, text_theta, number_theta
  )
  return {
      "table_datapoints_precision": (
          sum(score_dict["precision"]) / len(targets)
      ),
      "table_datapoints_recall": (
          sum(score_dict["recall"]) / len(targets)
      ),
      "table_datapoints_f1": sum(score_dict["f1"]) / len(targets),
  }

def evaluate_rms(generated_table,label_table):
  predictions=[generated_table]
  targets=[label_table]
  RMS = table_datapoints_precision_recall(targets, predictions)
  return RMS

def ko_deplot_convert_to_dataframe(generated_table_str):
    lines = generated_table_str.strip().split(" \n")
    headers=[]
    data=[]
    for i in range(len(lines[1].split(" | "))):
        headers.append(f"{i}")
    for line in lines[1:len(lines)-1]:
        data.append(line.split("| "))
    df = pd.DataFrame(data, columns=headers)
    return df

def ko_deplot_convert_to_dataframe2(label_table_str):
    lines = label_table_str.strip().split(" \n")
    headers=[]
    data=[]
    for i in range(len(lines[1].split(" | "))):
        headers.append(f"{i}")
    for line in lines[1:]:
        data.append(line.split("| "))
    df = pd.DataFrame(data, columns=headers)
    return df

def aihub_deplot_convert_to_dataframe(table_str):
    lines = table_str.strip().split("\n")
    headers = []
    if(len(lines[3].split(" | "))>len(lines[4].split(" | "))):
        category=lines[3].split(" | ")
        del category[0]
        value=lines[4].split(" | ")
        df=pd.DataFrame({"범례":category,"값":value})
        return df
    else:
        for i in range(len(lines[3].split(" | "))):
            headers.append(f"{i}")
        data = [line.split(" | ") for line in lines[3:]]
        df = pd.DataFrame(data, columns=headers)
        return df
def unichart_convert_to_dataframe(table_str):
    lines=table_str.split(" & ")
    headers=[]
    data=[]
    del lines[0]
    for i in range(len(lines[1].split(" | "))):
        headers.append(f"{i}")
    if lines[0]=="value":
        for line in lines[1:]:
            data.append(line.split(" | "))
    else:
        category=lines[0].split(" | ")
        category.insert(0," ")
        data.append(category)
        for line in lines[1:]:
            data.append(line.split(" | "))
    df=pd.DataFrame(data,columns=headers)
    return df

class Highlighter:
    def __init__(self):
        self.row = 0
        self.col = 0

    def compare_and_highlight(self, pred_table_elem, target_table, pred_table_row, props=''):
        if self.row >= pred_table_row:
            self.col += 1
            self.row = 0
        if pred_table_elem != target_table.iloc[self.row, self.col]:
            self.row += 1
            return props
        else:
            self.row += 1
            return None    

# 1. 데이터 로드
aihub_deplot_result_df = pd.read_csv('./aihub_deplot_result.csv')
ko_deplot_result= './ko-deplot-base-pred-epoch1-refinetuning.json'
unichart_result='./unichart_results.json'

# 2. 체크해야 하는 이미지 파일 로드
def load_image_checklist(file):
    with open(file, 'r') as f:
        #image_names = [f'"{line.strip()}"' for line in f]
        image_names = f.read().splitlines()
    return image_names

# 3. 현재 인덱스를 추적하기 위한 변수
current_index = 0
image_names = []
def show_image(current_idx):
    image_name=image_names[current_idx]
    image_path = f"./images/{image_name}.jpg"
    if not os.path.exists(image_path):
        raise FileNotFoundError(f"Image file not found: {image_path}")
    return Image.open(image_path)

# 4. 버튼 클릭 이벤트 핸들러
def non_real_time_check(file):
    highlighter1 = Highlighter()
    highlighter2 = Highlighter()
    highlighter3 = Highlighter()
    #global image_names, current_index
    #image_names = load_image_checklist(file)
    #current_index = 0
    #image=show_image(current_index)
    file_name =image_names[current_index].replace("Source","Label")

    json_path="./ko_deplot_labeling_data.json"
    with open(json_path, 'r', encoding='utf-8') as file:
        json_data = json.load(file)
    for key, value in json_data.items():
        if key == file_name:
            ko_deplot_labeling_str=value.get("txt").replace("<0x0A>","\n")
            ko_deplot_label_title=ko_deplot_labeling_str.split(" \n ")[0].replace("TITLE | ","제목:")
            break

    ko_deplot_rms_path="./ko_deplot_rms.txt"
    unichart_rms_path="./unichart_rms.txt"
    
    json_path="./unichart_labeling_data.json"
    with open(json_path, 'r', encoding='utf-8') as file:
        json_data = json.load(file)
    for entry in json_data:
        if entry["imgname"]==image_names[current_index]+".jpg":
            unichart_labeling_str=entry["label"]
            unichart_label_title=entry["label"].split(" & ")[0].split(" | ")[1]

    with open(ko_deplot_rms_path,'r',encoding='utf-8') as file:
        lines=file.readlines()
    flag=0
    for line in lines:
        parts=line.strip().split(", ")
        if(len(parts)==2 and parts[0]==image_names[current_index]):
            ko_deplot_rms=parts[1]
            flag=1
            break
    if(flag==0):
        ko_deplot_rms="none"
        
    with open(unichart_rms_path,'r',encoding='utf-8') as file:
        lines=file.readlines()
    flag=0
    for line in lines:
        parts=line.strip().split(": ")
        if(len(parts)==2 and parts[0]==image_names[current_index]+".jpg"):
            unichart_rms=parts[1]
            flag=1
            break
    if(flag==0):
        unichart_rms="none"
    
    
    
    ko_deplot_generated_title,ko_deplot_generated_table=ko_deplot_display_results(current_index)
    aihub_deplot_generated_table,aihub_deplot_label_table,aihub_deplot_generated_title,aihub_deplot_label_title=aihub_deplot_display_results(current_index)
    unichart_generated_table,unichart_generated_title=unichart_display_results(current_index)
    #ko_deplot_RMS=evaluate_rms(ko_deplot_generated_table,ko_deplot_labeling_str)
    aihub_deplot_RMS=evaluate_rms(aihub_deplot_generated_table,aihub_deplot_label_table)

    
    if flag == 1:
        value = [round(float(ko_deplot_rms), 1)]
    else:
        value = [0]

    ko_deplot_score_table = pd.DataFrame({
    'category': ['f1'],
    'value': value
    })
    
    value=[round(float(unichart_rms)/100,1)]
    unichart_score_table=pd.DataFrame({
        'category':['f1'],
        'value':value
    })
    aihub_deplot_score_table=pd.DataFrame({
        'category': ['precision', 'recall', 'f1'],
        'value': [
            round(aihub_deplot_RMS['table_datapoints_precision'],1),
            round(aihub_deplot_RMS['table_datapoints_recall'],1),
            round(aihub_deplot_RMS['table_datapoints_f1'],1)
        ]
    })
        
    ko_deplot_generated_df=ko_deplot_convert_to_dataframe(ko_deplot_generated_table)
    aihub_deplot_generated_df=aihub_deplot_convert_to_dataframe(aihub_deplot_generated_table)
    unichart_generated_df=unichart_convert_to_dataframe(unichart_generated_table)
    ko_deplot_labeling_df=ko_deplot_convert_to_dataframe2(ko_deplot_labeling_str)
    aihub_deplot_labeling_df=aihub_deplot_convert_to_dataframe(aihub_deplot_label_table)
    unichart_labeling_df=unichart_convert_to_dataframe(unichart_labeling_str)

    ko_deplot_generated_df_row=ko_deplot_generated_df.shape[0]
    aihub_deplot_generated_df_row=aihub_deplot_generated_df.shape[0]
    unichart_generated_df_row=unichart_generated_df.shape[0]


    styled_ko_deplot_table=ko_deplot_generated_df.style.applymap(highlighter1.compare_and_highlight,target_table=ko_deplot_labeling_df,pred_table_row=ko_deplot_generated_df_row,props='color:red')
    

    styled_aihub_deplot_table=aihub_deplot_generated_df.style.applymap(highlighter2.compare_and_highlight,target_table=aihub_deplot_labeling_df,pred_table_row=aihub_deplot_generated_df_row,props='color:red')


    styled_unichart_table=unichart_generated_df.style.applymap(highlighter3.compare_and_highlight,target_table=unichart_labeling_df,pred_table_row=unichart_generated_df_row,props='color:red')

    #return ko_deplot_convert_to_dataframe(ko_deplot_generated_table), aihub_deplot_convert_to_dataframe(aihub_deplot_generated_table), aihub_deplot_convert_to_dataframe(label_table), ko_deplot_score_table, aihub_deplot_score_table
    return gr.DataFrame(styled_ko_deplot_table,label=ko_deplot_generated_title+"(ko deplot 추론 결과)"),gr.DataFrame(styled_aihub_deplot_table,label=aihub_deplot_generated_title+"(aihub deplot 추론 결과)"),gr.DataFrame(styled_unichart_table,label="제목:"+unichart_generated_title+"(unichart 추론 결과)"),gr.DataFrame(ko_deplot_labeling_df,label=ko_deplot_label_title+"(ko deplot 정답 테이블)"), gr.DataFrame(aihub_deplot_labeling_df,label=aihub_deplot_label_title+"(aihub deplot 정답 테이블)"),gr.DataFrame(unichart_labeling_df,label="제목:"+unichart_label_title+"(unichart 정답 테이블)"),ko_deplot_score_table, aihub_deplot_score_table,unichart_score_table


def ko_deplot_display_results(index):
      filename=image_names[index]+".jpg"
      with open(ko_deplot_result, 'r', encoding='utf-8') as f:
        data = json.load(f)
      for entry in data:
        if entry['filename'].endswith(filename):
            #return entry['table']
            parts=entry['table'].split("\n",1)
            return parts[0].replace("TITLE | ","제목:"),entry['table']

def aihub_deplot_display_results(index):
    if index < 0 or index >= len(image_names):
        return "Index out of range", None, None
    image_name = image_names[index]
    image_row = aihub_deplot_result_df[aihub_deplot_result_df['data_id'] == image_name]
    if not image_row.empty:
        generated_table = image_row['generated_table'].values[0]
        generated_title=generated_table.split("\n")[1]
        label_table = image_row['label_table'].values[0]
        label_title=label_table.split("\n")[1]
        return generated_table, label_table, generated_title, label_title
    else:
        return "No results found for the image", None, None
def unichart_display_results(index):
    image_name=image_names[index]
    with open(unichart_result,'r',encoding='utf-8') as f:
        data=json.load(f)
    for entry in data:
        if entry['imgname']==image_name+".jpg":
            return entry['label'],entry['label'].split(" & ")[0].split(" | ")[1]

def previous_image():
    global current_index
    if current_index>0:
        current_index-=1
    image=show_image(current_index)
    return image, image_names[current_index],gr.update(interactive=current_index>0), gr.update(interactive=current_index<len(image_names)-1)
    
def next_image():
    global current_index
    if current_index<len(image_names)-1:
        current_index+=1
    image=show_image(current_index)
    return image, image_names[current_index],gr.update(interactive=current_index>0), gr.update(interactive=current_index<len(image_names)-1)

def real_time_check(image_file):
    highlighter1 = Highlighter()
    highlighter2 = Highlighter()
    highlighter3=Highlighter()
    image = Image.open(image_file)
    result_model1 = predict_model1(image)
    parts=result_model1.split("\n")
    del parts[-1]
    result_model1="\n".join(parts)
    ko_deplot_generated_title=result_model1.split("\n")[0].split(" | ")[1]
    ko_deplot_table=ko_deplot_convert_to_dataframe2(result_model1)

    result_model2 = predict_model2(image)
    aihub_deplot_generated_title=result_model2.split("\n")[1].split(":")[1]
    aihub_deplot_table=aihub_deplot_convert_to_dataframe(result_model2)
    image_base_name = os.path.basename(image_file.name).replace("Source","Label")
    file_name, _ = os.path.splitext(image_base_name)

    result_model3=predict_model3(image)
    unichart_table=unichart_convert_to_dataframe(result_model3)
    unichart_generated_title=result_model3.split(" & ")[0].split(" | ")[1]
    
    #aihub_labeling_data_json="./labeling_data/"+file_name+".json"
      
    json_path="./ko_deplot_labeling_data.json"    
    with open(json_path, 'r', encoding='utf-8') as file:   
        json_data = json.load(file)       
    for key, value in json_data.items():  
        if key == file_name:     
            ko_deplot_labeling_str=value.get("txt").replace("<0x0A>","\n")            
            ko_deplot_label_title=ko_deplot_labeling_str.split(" \n ")[0].split(" | ")[1]    
            break

    ko_deplot_label_table=ko_deplot_convert_to_dataframe2(ko_deplot_labeling_str)

    #aihub_deplot_labeling_str=process_json_file2(aihub_labeling_data_json)
    #aihub_deplot_label_title=aihub_deplot_labeling_str.split("\n")[1].split(":")[1]
    
    image_row = aihub_deplot_result_df[aihub_deplot_result_df['data_id'] == file_name.replace("Label","Source")]
    label_table=""
    label_title=""
    if not image_row.empty:
        label_table = image_row['label_table'].values[0]
        label_title=label_table.split("\n")[1]
    
    aihub_deplot_label_table=aihub_deplot_convert_to_dataframe(label_table)

    json_path="./unichart_labeling_data.json"
    with open(json_path, 'r', encoding='utf-8') as file:
        json_data = json.load(file)
    for entry in json_data:
        if entry["imgname"]==os.path.basename(image_file.name):
            unichart_labeling_str=entry["label"]
            unichart_label_title=entry["label"].split(" & ")[0].split(" | ")[1]
    unichart_label_table=unichart_convert_to_dataframe(unichart_labeling_str)

    ko_deplot_RMS=evaluate_rms(result_model1,ko_deplot_labeling_str)
    aihub_deplot_RMS=evaluate_rms(result_model2,label_table)
    unichart_RMS=evaluate_rms(result_model3.replace("Characteristic","Title").replace("&","\n"),unichart_labeling_str.replace("Characteristic","Title").replace("&","\n"))
    ko_deplot_score_table=pd.DataFrame({
    'category': ['precision', 'recall', 'f1'],
    'value': [
        round(ko_deplot_RMS['table_datapoints_precision'],1),
        round(ko_deplot_RMS['table_datapoints_recall'],1),
        round(ko_deplot_RMS['table_datapoints_f1'],1)
    ]
})
    aihub_deplot_score_table=pd.DataFrame({
        'category': ['precision', 'recall', 'f1'],
        'value': [
            round(aihub_deplot_RMS['table_datapoints_precision'],1),
            round(aihub_deplot_RMS['table_datapoints_recall'],1),
            round(aihub_deplot_RMS['table_datapoints_f1'],1)
        ]
        })
    
    unichart_score_table=pd.DataFrame({
        'category': ['precision', 'recall', 'f1'],
        'value': [
            round(unichart_RMS['table_datapoints_precision'],1),
            round(unichart_RMS['table_datapoints_recall'],1),
            round(unichart_RMS['table_datapoints_f1'],1)
        ]
        })
    
    ko_deplot_generated_df_row=ko_deplot_table.shape[0]
    aihub_deplot_generated_df_row=aihub_deplot_table.shape[0]
    unichart_generated_df_row=unichart_table.shape[0]
    styled_ko_deplot_table=ko_deplot_table.style.applymap(highlighter1.compare_and_highlight,target_table=ko_deplot_label_table,pred_table_row=ko_deplot_generated_df_row,props='color:red')
    styled_aihub_deplot_table=aihub_deplot_table.style.applymap(highlighter2.compare_and_highlight,target_table=aihub_deplot_label_table,pred_table_row=aihub_deplot_generated_df_row,props='color:red')
    styled_unichart_table=unichart_table.style.applymap(highlighter3.compare_and_highlight,target_table=unichart_label_table,pred_table_row=unichart_generated_df_row,props='color:red')
    return gr.DataFrame(styled_ko_deplot_table,label=ko_deplot_generated_title+"(kodeplot 추론 결과)") , gr.DataFrame(styled_aihub_deplot_table,label=aihub_deplot_generated_title+"(aihub deplot 추론 결과)"),gr.DataFrame(styled_unichart_table,label=unichart_generated_title+"(unichart 추론 결과)"),gr.DataFrame(ko_deplot_label_table,label=ko_deplot_label_title+"(kodeplot 정답 테이블)"),gr.DataFrame(aihub_deplot_label_table,label=label_title+"(aihub deplot 정답 테이블)"),gr.DataFrame(unichart_label_table,label=unichart_label_title+"(unichart 정답 테이블)"),ko_deplot_score_table, aihub_deplot_score_table,unichart_score_table
    #return ko_deplot_table,aihub_deplot_table,aihub_deplot_label_table,ko_deplot_score_table,aihub_deplot_score_table
def inference(mode,image_uploader,file_uploader):
    if(mode=="이미지 업로드"):
        ko_deplot_table, aihub_deplot_table, unichart_table, ko_deplot_label_table,aihub_deplot_label_table,unichart_label_table,ko_deplot_score_table, aihub_deplot_score_table,unichart_score_table= real_time_check(image_uploader)
        return ko_deplot_table, aihub_deplot_table, unichart_table,ko_deplot_label_table, aihub_deplot_label_table,unichart_label_table,ko_deplot_score_table, aihub_deplot_score_table,unichart_score_table
    else:
        styled_ko_deplot_table,styled_aihub_deplot_table,styled_unichart_table,ko_deplot_label_table,aihub_deplot_label_table,unichart_label_table,ko_deplot_score_table,aihub_deplot_score_table, unichart_score_table=non_real_time_check(file_uploader)
        return styled_ko_deplot_table, styled_aihub_deplot_table, styled_unichart_table,ko_deplot_label_table,aihub_deplot_label_table,unichart_label_table,ko_deplot_score_table, aihub_deplot_score_table, unichart_score_table
def interface_selector(selector):
    if selector == "이미지 업로드":
        return gr.update(visible=True),gr.update(visible=False),gr.State("image_upload"),gr.update(visible=False),gr.update(visible=False)
    elif selector == "파일 업로드":
        return gr.update(visible=False),gr.update(visible=True),gr.State("file_upload"), gr.update(visible=True),gr.update(visible=True)

def file_selector(selector):
    if selector == "low score 차트":
        return gr.File("./new_bottom_20_percent_images.txt")
    elif selector == "high score 차트":
        return gr.File("./new_top_20_percent_images.txt")

def update_results(model_type):
    if "ko_deplot" == model_type:
        return gr.update(visible=True),gr.update(visible=True),gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),gr.update(visible=True),gr.update(visible=False),gr.update(visible=False)
    elif "aihub_deplot" == model_type:
        return gr.update(visible=False),gr.update(visible=False),gr.update(visible=True),gr.update(visible=True),gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),gr.update(visible=True),gr.update(visible=False)
    elif "unichart"==model_type:
        return gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),gr.update(visible=True),gr.update(visible=True),gr.update(visible=False),gr.update(visible=False),gr.update(visible=True)
    else:
        return gr.update(visible=True), gr.update(visible=True),gr.update(visible=True),gr.update(visible=True),gr.update(visible=True),gr.update(visible=True),gr.update(visible=True),gr.update(visible=True),gr.update(visible=True)

def display_image(image_file):
    image=Image.open(image_file)
    return image, os.path.basename(image_file)

def display_image_in_file(image_checklist):
    global image_names, current_index
    image_names = load_image_checklist(image_checklist)
    image=show_image(current_index)
    return image,image_names[current_index]

def update_file_based_on_chart_type(chart_type, all_file_path):
    with open(all_file_path, 'r', encoding='utf-8') as file:
        lines = file.readlines()
    filtered_lines=[] 
    if chart_type == "전체":
        filtered_lines = lines
    elif chart_type == "일반 가로 막대형":
        filtered_lines = [line for line in lines if "_horizontal bar_standard" in line]
    elif chart_type=="누적 가로 막대형":
        filtered_lines = [line for line in lines if "_horizontal bar_accumulation" in line]     
    elif chart_type=="100% 기준 누적 가로 막대형":
        filtered_lines = [line for line in lines if "_horizontal bar_100per accumulation" in line] 
    elif chart_type=="일반 세로 막대형":
        filtered_lines = [line for line in lines if "_vertical bar_standard" in line]
    elif chart_type=="누적 세로 막대형":
        filtered_lines = [line for line in lines if "_vertical bar_accumulation" in line]     
    elif chart_type=="100% 기준 누적 세로 막대형":
        filtered_lines = [line for line in lines if "_vertical bar_100per accumulation" in line]
    elif chart_type=="선형":
        filtered_lines = [line for line in lines if "_line_standard" in line]
    elif chart_type=="원형":
        filtered_lines = [line for line in lines if "_pie_standard" in line]
    elif chart_type=="기타 방사형":
        filtered_lines = [line for line in lines if "_etc_radial" in line]     
    elif chart_type=="기타 혼합형":
        filtered_lines = [line for line in lines if "_etc_mix" in line]
    # 새로운 파일에 기록
    new_file_path = "./filtered_chart_images.txt"
    with open(new_file_path, 'w', encoding='utf-8') as file:
        file.writelines(filtered_lines)

    return new_file_path

def handle_chart_type_change(chart_type,all_file_path):
    new_file_path = update_file_based_on_chart_type(chart_type, all_file_path)
    global image_names, current_index
    image_names = load_image_checklist(new_file_path)
    current_index=0
    image=show_image(current_index)
    return image,image_names[current_index]

with gr.Blocks() as iface:
    mode=gr.State("image_upload")
    with gr.Row():
        with gr.Column():
            #mode_label=gr.Text("이미지 업로드가 선택되었습니다.")
            upload_option = gr.Radio(choices=["이미지 업로드", "파일 업로드"], value="이미지 업로드", label="업로드 옵션")
            #with gr.Row():
                #image_button = gr.Button("이미지 업로드")
                #file_button = gr.Button("파일 업로드")

            # 이미지와 파일 업로드 컴포넌트 (초기에는 숨김 상태)
            # global image_uploader,file_uploader
            image_uploader= gr.File(file_count="single",file_types=["image"],visible=True)
            file_uploader= gr.File(file_count="single", file_types=[".txt"], visible=False)
            file_upload_option=gr.Radio(choices=["low score 차트","high score 차트"],label="파일 업로드 옵션",visible=False)
            chart_type = gr.Dropdown(["일반 가로 막대형","누적 가로 막대형","100% 기준 누적 가로 막대형", "일반 세로 막대형","누적 세로 막대형","100% 기준 누적 세로 막대형","선형", "원형", "기타 방사형", "기타 혼합형", "전체"], label="Chart Type", value="all")
            model_type=gr.Dropdown(["ko_deplot","aihub_deplot","unichart","all"],label="model")
            image_displayer=gr.Image(visible=True)
            with gr.Row():
                pre_button=gr.Button("이전",interactive="False")
                next_button=gr.Button("다음")
            image_name=gr.Text("이미지 이름",visible=False)
            #image_button.click(interface_selector, inputs=gr.State("이미지 업로드"), outputs=[image_uploader,file_uploader,mode,mode_label,image_name])
            #file_button.click(interface_selector, inputs=gr.State("파일 업로드"), outputs=[image_uploader, file_uploader,mode,mode_label,image_name])
            inference_button=gr.Button("추론")
        with gr.Column():
            ko_deplot_generated_table=gr.DataFrame(visible=False,label="ko-deplot 추론 결과")
            aihub_deplot_generated_table=gr.DataFrame(visible=False,label="aihub-deplot 추론 결과")
            unichart_generated_table=gr.DataFrame(visible=False,label="unichart 추론 결과")
        with gr.Column():
            ko_deplot_label_table=gr.DataFrame(visible=False,label="ko-deplot 정답테이블")
            aihub_deplot_label_table=gr.DataFrame(visible=False,label="aihub-deplot 정답테이블")
            unichart_label_table=gr.DataFrame(visible=False,label="unichart 정답테이블")
        with gr.Column():
            ko_deplot_score_table=gr.DataFrame(visible=False,label="ko_deplot 점수")
            aihub_deplot_score_table=gr.DataFrame(visible=False,label="aihub_deplot 점수")
            unichart_score_table=gr.DataFrame(visible=False,label="unichart 점수")
    model_type.change(
                        update_results,
                        inputs=[model_type],
                        outputs=[ko_deplot_generated_table,ko_deplot_score_table,aihub_deplot_generated_table,aihub_deplot_score_table,unichart_generated_table,unichart_score_table,ko_deplot_label_table,aihub_deplot_label_table,unichart_label_table]
                        )  
    
    upload_option.change(
        interface_selector,
        inputs=[upload_option],
        outputs=[image_uploader, file_uploader, mode, image_name,file_upload_option]
    )

    file_upload_option.change(
        file_selector,
        inputs=[file_upload_option],
        outputs=[file_uploader]
    )

    chart_type.change(handle_chart_type_change, inputs=[chart_type,file_uploader],outputs=[image_displayer,image_name])
    image_uploader.upload(display_image,inputs=[image_uploader],outputs=[image_displayer,image_name])
    file_uploader.change(display_image_in_file,inputs=[file_uploader],outputs=[image_displayer,image_name])
    pre_button.click(previous_image, outputs=[image_displayer,image_name,pre_button,next_button])
    next_button.click(next_image, outputs=[image_displayer,image_name,pre_button,next_button])
    inference_button.click(inference,inputs=[upload_option,image_uploader,file_uploader],outputs=[ko_deplot_generated_table, aihub_deplot_generated_table, unichart_generated_table, ko_deplot_label_table, aihub_deplot_label_table, unichart_label_table, ko_deplot_score_table, aihub_deplot_score_table,unichart_score_table])

if __name__ == "__main__":
    print("Launching Gradio interface...")
    sys.stdout.flush()  # stdout 버퍼를 비웁니다.
    iface.launch(share=True)
    time.sleep(2)  # Gradio URL이 출력될 때까지 잠시 기다립니다.
    sys.stdout.flush()  # 다시 stdout 버퍼를 비웁니다.
        # Gradio가 제공하는 URLs을 파일에 기록합니다.
    with open("gradio_url.log", "w") as f:
        print(iface.local_url, file=f)
        print(iface.share_url, file=f)