mishig HF staff commited on
Commit
88afc1d
1 Parent(s): eec5743

fetch tokenizers data with token

Browse files
Files changed (1) hide show
  1. src/routes/+page.server.ts +6 -1
src/routes/+page.server.ts CHANGED
@@ -17,8 +17,13 @@ export const load: PageServerLoad = async ({ fetch }) => {
17
 
18
  const promises = compatibleModels.map(async model => {
19
  const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
20
- const res = await fetch(configUrl);
 
 
 
 
21
  if (!res.ok) {
 
22
  return null; // Ignore failed requests by returning null
23
  }
24
  const tokenizerConfig = await res.json();
 
17
 
18
  const promises = compatibleModels.map(async model => {
19
  const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
20
+ const res = await fetch(configUrl, {
21
+ headers: {
22
+ Authorization: `Bearer ${HF_TOKEN}`,
23
+ },
24
+ });
25
  if (!res.ok) {
26
+ console.error("Error fetching tokenizer file", res.status, res.statusText);
27
  return null; // Ignore failed requests by returning null
28
  }
29
  const tokenizerConfig = await res.json();