update

src/submission/check_validity.py

@@ -7,11 +7,14 @@ from datetime import datetime, timedelta, timezone
 import huggingface_hub
 from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
+
+from transformers import AutoConfig, AutoTokenizer
 from transformers.models.auto.tokenization_auto import tokenizer_class_from_name, get_tokenizer_config
 
 from src.envs import HAS_HIGHER_RATE_LIMIT
 
+from typing import Optional
+
 
 # ht to @Wauplin, thank you for the snippet!
 # See https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/317
@@ -37,39 +40,23 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
     return True, ""
 
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, Optional[str], Optional[AutoConfig]]:
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) #, force_download=True)
         if test_tokenizer:
-            tokenizer_config = get_tokenizer_config(model_name)
-
-            if tokenizer_config is not None:
-                tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
-            else:
-                tokenizer_class_candidate = config.tokenizer_class
-
-            tokenizer_class = None
-            if tokenizer_class_candidate is not None:
-                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
-
-            if tokenizer_class is None:
-                return (
-                    False,
-                    f"uses {tokenizer_class_candidate}, which is not in a transformers release, therefore not supported at the moment.",
-                    None
-                )
+            try:
+                AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return False, f"uses a tokenizer which is not in a transformers release: {e}", None
+            except Exception as e:
+                return False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None
         return True, None, config
 
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
+    except ValueError as e:
+        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", None
 
     except Exception as e:
-        print('XXX', e)
-        return False, "was not found on hub!", None
+        return False, f"was not found on hub -- {str(e)}", None
 
 
 def get_model_size(model_info: ModelInfo, precision: str):