Update utils/download_url.py
Browse files- utils/download_url.py +0 -61
utils/download_url.py
CHANGED
@@ -5,67 +5,6 @@ from torch.hub import download_url_to_file, get_dir
|
|
5 |
from tqdm import tqdm
|
6 |
from urllib.parse import urlparse
|
7 |
|
8 |
-
from .misc import sizeof_fmt
|
9 |
-
|
10 |
-
|
11 |
-
def download_file_from_google_drive(file_id, save_path):
|
12 |
-
"""Download files from google drive.
|
13 |
-
|
14 |
-
Ref:
|
15 |
-
https://stackoverflow.com/questions/25010369/wget-curl-large-file-from-google-drive # noqa E501
|
16 |
-
|
17 |
-
Args:
|
18 |
-
file_id (str): File id.
|
19 |
-
save_path (str): Save path.
|
20 |
-
"""
|
21 |
-
|
22 |
-
session = requests.Session()
|
23 |
-
URL = 'https://docs.google.com/uc?export=download'
|
24 |
-
params = {'id': file_id}
|
25 |
-
|
26 |
-
response = session.get(URL, params=params, stream=True)
|
27 |
-
token = get_confirm_token(response)
|
28 |
-
if token:
|
29 |
-
params['confirm'] = token
|
30 |
-
response = session.get(URL, params=params, stream=True)
|
31 |
-
|
32 |
-
# get file size
|
33 |
-
response_file_size = session.get(URL, params=params, stream=True, headers={'Range': 'bytes=0-2'})
|
34 |
-
if 'Content-Range' in response_file_size.headers:
|
35 |
-
file_size = int(response_file_size.headers['Content-Range'].split('/')[1])
|
36 |
-
else:
|
37 |
-
file_size = None
|
38 |
-
|
39 |
-
save_response_content(response, save_path, file_size)
|
40 |
-
|
41 |
-
|
42 |
-
def get_confirm_token(response):
|
43 |
-
for key, value in response.cookies.items():
|
44 |
-
if key.startswith('download_warning'):
|
45 |
-
return value
|
46 |
-
return None
|
47 |
-
|
48 |
-
|
49 |
-
def save_response_content(response, destination, file_size=None, chunk_size=32768):
|
50 |
-
if file_size is not None:
|
51 |
-
pbar = tqdm(total=math.ceil(file_size / chunk_size), unit='chunk')
|
52 |
-
|
53 |
-
readable_file_size = sizeof_fmt(file_size)
|
54 |
-
else:
|
55 |
-
pbar = None
|
56 |
-
|
57 |
-
with open(destination, 'wb') as f:
|
58 |
-
downloaded_size = 0
|
59 |
-
for chunk in response.iter_content(chunk_size):
|
60 |
-
downloaded_size += chunk_size
|
61 |
-
if pbar is not None:
|
62 |
-
pbar.update(1)
|
63 |
-
pbar.set_description(f'Download {sizeof_fmt(downloaded_size)} / {readable_file_size}')
|
64 |
-
if chunk: # filter out keep-alive new chunks
|
65 |
-
f.write(chunk)
|
66 |
-
if pbar is not None:
|
67 |
-
pbar.close()
|
68 |
-
|
69 |
|
70 |
def load_file_from_url(url, model_dir=None, progress=True, file_name=None):
|
71 |
"""Load file form http url, will download models if necessary.
|
|
|
5 |
from tqdm import tqdm
|
6 |
from urllib.parse import urlparse
|
7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
def load_file_from_url(url, model_dir=None, progress=True, file_name=None):
|
10 |
"""Load file form http url, will download models if necessary.
|