davanstrien HF staff commited on
Commit
10fd16b
1 Parent(s): faa9e8b

Add code to create 'models' directory for caching fasttext model

Browse files
Files changed (1) hide show
  1. main.py +4 -3
main.py CHANGED
@@ -11,6 +11,7 @@ from huggingface_hub.utils import logging
11
  from toolz import concat, groupby, valmap
12
  from fastapi import FastAPI
13
  from httpx import AsyncClient
 
14
 
15
  app = FastAPI()
16
  logger = logging.get_logger(__name__)
@@ -141,13 +142,13 @@ def yield_clean_rows(rows: Union[list[str], str], min_length: int = 3) -> Iterat
141
  FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
142
 
143
  # model = load_model(DEFAULT_FAST_TEXT_MODEL)
144
-
145
  model = fasttext.load_model(
146
  hf_hub_download(
147
  "facebook/fasttext-language-identification",
148
  "model.bin",
149
- cache_dir=".",
150
- local_dir=".",
151
  local_dir_use_symlinks=False,
152
  )
153
  )
 
11
  from toolz import concat, groupby, valmap
12
  from fastapi import FastAPI
13
  from httpx import AsyncClient
14
+ from pathlib import Path
15
 
16
  app = FastAPI()
17
  logger = logging.get_logger(__name__)
 
142
  FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
143
 
144
  # model = load_model(DEFAULT_FAST_TEXT_MODEL)
145
+ Path("models").mkdir(parents=True, exist_ok=True)
146
  model = fasttext.load_model(
147
  hf_hub_download(
148
  "facebook/fasttext-language-identification",
149
  "model.bin",
150
+ cache_dir="models",
151
+ local_dir="models",
152
  local_dir_use_symlinks=False,
153
  )
154
  )