Commit
•
10fd16b
1
Parent(s):
faa9e8b
Add code to create 'models' directory for caching fasttext model
Browse files
main.py
CHANGED
@@ -11,6 +11,7 @@ from huggingface_hub.utils import logging
|
|
11 |
from toolz import concat, groupby, valmap
|
12 |
from fastapi import FastAPI
|
13 |
from httpx import AsyncClient
|
|
|
14 |
|
15 |
app = FastAPI()
|
16 |
logger = logging.get_logger(__name__)
|
@@ -141,13 +142,13 @@ def yield_clean_rows(rows: Union[list[str], str], min_length: int = 3) -> Iterat
|
|
141 |
FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
|
142 |
|
143 |
# model = load_model(DEFAULT_FAST_TEXT_MODEL)
|
144 |
-
|
145 |
model = fasttext.load_model(
|
146 |
hf_hub_download(
|
147 |
"facebook/fasttext-language-identification",
|
148 |
"model.bin",
|
149 |
-
cache_dir="
|
150 |
-
local_dir="
|
151 |
local_dir_use_symlinks=False,
|
152 |
)
|
153 |
)
|
|
|
11 |
from toolz import concat, groupby, valmap
|
12 |
from fastapi import FastAPI
|
13 |
from httpx import AsyncClient
|
14 |
+
from pathlib import Path
|
15 |
|
16 |
app = FastAPI()
|
17 |
logger = logging.get_logger(__name__)
|
|
|
142 |
FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
|
143 |
|
144 |
# model = load_model(DEFAULT_FAST_TEXT_MODEL)
|
145 |
+
Path("models").mkdir(parents=True, exist_ok=True)
|
146 |
model = fasttext.load_model(
|
147 |
hf_hub_download(
|
148 |
"facebook/fasttext-language-identification",
|
149 |
"model.bin",
|
150 |
+
cache_dir="models",
|
151 |
+
local_dir="models",
|
152 |
local_dir_use_symlinks=False,
|
153 |
)
|
154 |
)
|