Spaces:
Running
Running
upload to community org
Browse files
app.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
import pathlib
|
2 |
import random
|
3 |
import string
|
@@ -100,6 +101,11 @@ This Space is heavily inspired by LazyMergeKit by Maxime Labonne (see [Colab](ht
|
|
100 |
|
101 |
examples = [[str(f)] for f in pathlib.Path("examples").glob("*.yml")]
|
102 |
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
def merge(yaml_config: str, hf_token: str, repo_name: str) -> Iterable[List[Log]]:
|
105 |
runner = LogsViewRunner()
|
@@ -113,9 +119,22 @@ def merge(yaml_config: str, hf_token: str, repo_name: str) -> Iterable[List[Log]
|
|
113 |
yield runner.log(f"Invalid yaml {e}", level="ERROR")
|
114 |
return
|
115 |
|
|
|
116 |
if not hf_token:
|
117 |
-
|
118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
119 |
api = huggingface_hub.HfApi(token=hf_token)
|
120 |
|
121 |
with tempfile.TemporaryDirectory() as tmpdirname:
|
@@ -133,6 +152,9 @@ def merge(yaml_config: str, hf_token: str, repo_name: str) -> Iterable[List[Log]
|
|
133 |
repo_name += "-" + "".join(random.choices(string.ascii_lowercase, k=7))
|
134 |
repo_name = repo_name.replace("/", "-").strip("-")
|
135 |
|
|
|
|
|
|
|
136 |
try:
|
137 |
yield runner.log(f"Creating repo {repo_name}")
|
138 |
repo_url = api.create_repo(repo_name, exist_ok=True)
|
@@ -169,7 +191,7 @@ with gr.Blocks() as demo:
|
|
169 |
label="HF Write Token",
|
170 |
info="https://hf.co/settings/token",
|
171 |
type="password",
|
172 |
-
placeholder="
|
173 |
)
|
174 |
repo_name = gr.Textbox(
|
175 |
lines=1,
|
|
|
1 |
+
import os
|
2 |
import pathlib
|
3 |
import random
|
4 |
import string
|
|
|
101 |
|
102 |
examples = [[str(f)] for f in pathlib.Path("examples").glob("*.yml")]
|
103 |
|
104 |
+
# Do not set community token as `HF_TOKEN` to avoid accidentally using it in merge scripts.
|
105 |
+
# `COMMUNITY_HF_TOKEN` is used to upload models to the community organization (https://huggingface.co/mergekit-community)
|
106 |
+
# when user do not provide a token.
|
107 |
+
COMMUNITY_HF_TOKEN = os.getenv("COMMUNITY_HF_TOKEN")
|
108 |
+
|
109 |
|
110 |
def merge(yaml_config: str, hf_token: str, repo_name: str) -> Iterable[List[Log]]:
|
111 |
runner = LogsViewRunner()
|
|
|
119 |
yield runner.log(f"Invalid yaml {e}", level="ERROR")
|
120 |
return
|
121 |
|
122 |
+
is_community_model = False
|
123 |
if not hf_token:
|
124 |
+
if "/" in repo_name and not repo_name.startswith("mergekit-community/"):
|
125 |
+
yield runner.log(
|
126 |
+
f"Cannot upload merge model to namespace {repo_name.split('/')[0]}: you must provide a valid token.",
|
127 |
+
level="ERROR",
|
128 |
+
)
|
129 |
+
return
|
130 |
+
yield runner.log(
|
131 |
+
"No HF token provided. Your merged model will be uploaded to the https://huggingface.co/mergekit-community organization."
|
132 |
+
)
|
133 |
+
is_community_model = True
|
134 |
+
if not COMMUNITY_HF_TOKEN:
|
135 |
+
raise gr.Error("Cannot upload to community org: community token not set by Space owner.")
|
136 |
+
hf_token = COMMUNITY_HF_TOKEN
|
137 |
+
|
138 |
api = huggingface_hub.HfApi(token=hf_token)
|
139 |
|
140 |
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
|
152 |
repo_name += "-" + "".join(random.choices(string.ascii_lowercase, k=7))
|
153 |
repo_name = repo_name.replace("/", "-").strip("-")
|
154 |
|
155 |
+
if is_community_model and not repo_name.startswith("mergekit-community/"):
|
156 |
+
repo_name = f"mergekit-community/{repo_name}"
|
157 |
+
|
158 |
try:
|
159 |
yield runner.log(f"Creating repo {repo_name}")
|
160 |
repo_url = api.create_repo(repo_name, exist_ok=True)
|
|
|
191 |
label="HF Write Token",
|
192 |
info="https://hf.co/settings/token",
|
193 |
type="password",
|
194 |
+
placeholder="Optional. Will upload merged model to MergeKit Community if empty.",
|
195 |
)
|
196 |
repo_name = gr.Textbox(
|
197 |
lines=1,
|