Nanobit commited on
Commit
b832a0a
1 Parent(s): afb31e1

Black formatting

Browse files
src/axolotl/utils/data.py CHANGED
@@ -107,7 +107,9 @@ def load_tokenized_prepared_datasets(
107
  use_auth_token=use_auth_token,
108
  )
109
  else:
110
- ds: Dataset = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
 
 
111
  else:
112
  fp = hf_hub_download(
113
  repo_id=d.path, repo_type="dataset", filename=d.data_files
@@ -293,7 +295,8 @@ def load_prepare_datasets(
293
  f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
294
  )
295
  dataset = load_dataset(
296
- f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
 
297
  )
298
  dataset = dataset["train"]
299
  except Exception: # pylint: disable=broad-except
 
107
  use_auth_token=use_auth_token,
108
  )
109
  else:
110
+ ds: Dataset = load_dataset(
111
+ d.path, streaming=False, use_auth_token=use_auth_token
112
+ )
113
  else:
114
  fp = hf_hub_download(
115
  repo_id=d.path, repo_type="dataset", filename=d.data_files
 
295
  f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
296
  )
297
  dataset = load_dataset(
298
+ f"{cfg.push_dataset_to_hub}/{ds_hash}",
299
+ use_auth_token=use_auth_token,
300
  )
301
  dataset = dataset["train"]
302
  except Exception: # pylint: disable=broad-except
src/axolotl/utils/validation.py CHANGED
@@ -40,7 +40,9 @@ def validate_config(cfg):
40
  )
41
 
42
  if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
43
- raise ValueError("Require cfg.hf_use_auth_token to be True for push_dataset_to_hub")
 
 
44
 
45
  # TODO
46
  # MPT 7b
 
40
  )
41
 
42
  if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
43
+ raise ValueError(
44
+ "Require cfg.hf_use_auth_token to be True for push_dataset_to_hub"
45
+ )
46
 
47
  # TODO
48
  # MPT 7b
tests/test_validation.py CHANGED
@@ -117,4 +117,3 @@ class ValidationTest(unittest.TestCase):
117
  }
118
  )
119
  validate_config(cfg)
120
-
 
117
  }
118
  )
119
  validate_config(cfg)