:hammer_and_pick: Looks ok

app.py

@@ -2,12 +2,15 @@ import os
 import random
 
 import huggingface_hub
+import numpy as np
 from datasets import load_dataset, Dataset
 from dotenv import load_dotenv
 from pytorch_lightning import LightningDataModule
 from pytorch_lightning.utilities.types import TRAIN_DATALOADERS, EVAL_DATALOADERS
 from torch.utils.data import DataLoader, IterableDataset
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score, roc_auc_score
+# from torchmetrics.classification import BinaryAccuracy, BinaryAUROC, BinaryF1Score, BinaryPrecision, BinaryRecall
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoModel
 from transformers import TrainingArguments, Trainer
 import torch
 import logging
@@ -200,12 +203,41 @@ def login_inside_huggingface_virtualmachine():
   pass
 
 
+# use sklearn cz torchmetrics.classification gave array index out of bound exception :/ (whatever it is called in python)
+def compute_metrics_using_sklearn(p):
+  try:
+    pred, labels = p
+
+    # Get predicted class labels
+    pred_labels = np.argmax(pred, axis=1)
+
+    # Get predicted probabilities for the positive class
+    pred_probs = pred[:, 1]  # Assuming binary classification and 2 output classes
+
+    accuracy = accuracy_score(y_true=labels, y_pred=pred_labels)
+    recall = recall_score(y_true=labels, y_pred=pred_labels)
+    precision = precision_score(y_true=labels, y_pred=pred_labels)
+    f1 = f1_score(y_true=labels, y_pred=pred_labels)
+    roc_auc = roc_auc_score(y_true=labels, y_score=pred_probs)
+
+    return {"accuracy": accuracy, "roc_auc": roc_auc, "precision": precision, "recall": recall, "f1": f1}
+
+  except Exception as x:
+    print(f"compute_metrics_using_sklearn failed with exception: {x}")
+    return {"accuracy": 0, "roc_auc": 0, "precision": 0, "recall": 0, "f1": 0}
+
+
 def start():
   os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
 
   login_inside_huggingface_virtualmachine()
   WINDOW = 4000
   batch_size = 100
+  model_local_directory = f"my-awesome-model-{WINDOW}"
+  model_remote_repository = f"fahimfarhan/hyenadna-sm-32k-mqtl-classifier-seq-len-{WINDOW}"
+
+  is_my_laptop = os.path.isfile("/home/soumic/Codes/mqtl-classification/src/inputdata/dataset_4000_train_binned.csv")
+
   tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME, trust_remote_code=True)
   classifier_model = AutoModelForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME,
                                                                         torch_dtype=torch.bfloat16,
@@ -256,6 +288,7 @@ def start():
     args=training_args,
     train_dataset=train_ds,
     eval_dataset=val_ds,
+    compute_metrics=compute_metrics_using_sklearn  # torch_metrics.compute_metrics
   )
   # train, and validate
   result = trainer.train()
@@ -274,11 +307,8 @@ def start():
   finally:
     # save the model
     model_name = "HyenaDnaMQtlClassifier"
-    is_my_laptop = os.path.isfile("/home/soumic/Codes/mqtl-classification/src/inputdata/dataset_4000_train_binned.csv")
-    model_repository_name = f"fahimfarhan/hyenadna-sm-32k-mqtl-classifier-seq-len-{WINDOW}"
 
-    model_subdirectory = f"my-awesome-model-{WINDOW}"
-    classifier_model.save_pretrained(save_directory=model_subdirectory, safe_serialization=False)
+    classifier_model.save_pretrained(save_directory=model_local_directory, safe_serialization=False)
 
     # push to the hub
     commit_message = f":tada: Push model for window size {WINDOW} from huggingface space"
@@ -286,7 +316,7 @@ def start():
       commit_message = f":tada: Push model for window size {WINDOW} from zephyrus"
 
     classifier_model.push_to_hub(
-      repo_id=model_repository_name,
+      repo_id=model_remote_repository,
       # subfolder=f"my-awesome-model-{WINDOW}", subfolder didn't work :/
       commit_message=commit_message,  # f":tada: Push model for window size {WINDOW}"
       safe_serialization=False
@@ -294,6 +324,20 @@ def start():
   pass
 
 
+def interprete_demo():
+  is_my_laptop = True
+  WINDOW = 4000
+  batch_size = 100
+  model_local_directory = f"my-awesome-model-{WINDOW}"
+  model_remote_repository = f"fahimfarhan/hyenadna-sm-32k-mqtl-classifier-seq-len-{WINDOW}"
+
+  try:
+    classifier_model = AutoModel.from_pretrained(model_remote_repository)
+    # todo: use captum / gentech-grelu to interpret the model
+  except Exception as x:
+    print(x)
+
+
 if __name__ == '__main__':
   start()
   pass