Spaces:
Runtime error
Runtime error
zetavg
commited on
Commit
•
28664d2
1
Parent(s):
6947876
show training speed
Browse files
llama_lora/ui/finetune/style.css
CHANGED
@@ -315,6 +315,7 @@
|
|
315 |
padding: var(--size-1) var(--size-2);
|
316 |
font-size: var(--text-sm);
|
317 |
font-family: var(--font-mono);
|
|
|
318 |
}
|
319 |
#finetune_training_status .progress-block .status {
|
320 |
white-space: pre-wrap;
|
@@ -328,6 +329,7 @@
|
|
328 |
z-index: var(--layer-2);
|
329 |
width: var(--size-full);
|
330 |
padding: 8px 0;
|
|
|
331 |
}
|
332 |
#finetune_training_status .progress-block .progress-level-inner {
|
333 |
margin: var(--size-2) auto;
|
|
|
315 |
padding: var(--size-1) var(--size-2);
|
316 |
font-size: var(--text-sm);
|
317 |
font-family: var(--font-mono);
|
318 |
+
text-align: right;
|
319 |
}
|
320 |
#finetune_training_status .progress-block .status {
|
321 |
white-space: pre-wrap;
|
|
|
329 |
z-index: var(--layer-2);
|
330 |
width: var(--size-full);
|
331 |
padding: 8px 0;
|
332 |
+
text-align: center;
|
333 |
}
|
334 |
#finetune_training_status .progress-block .progress-level-inner {
|
335 |
margin: var(--size-2) auto;
|
llama_lora/ui/finetune/training.py
CHANGED
@@ -398,10 +398,16 @@ def render_training_status():
|
|
398 |
if time_remaining >= 0:
|
399 |
meta_info.append(
|
400 |
f"{format_time(time_elapsed)}<{format_time(time_remaining)}")
|
401 |
-
meta_info.append(f"ETA: {format_timestamp(Global.training_eta)}")
|
402 |
else:
|
403 |
meta_info.append(format_time(time_elapsed))
|
404 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
405 |
params_info_html = ""
|
406 |
if Global.training_params_info_text:
|
407 |
params_info_html = f"""
|
|
|
398 |
if time_remaining >= 0:
|
399 |
meta_info.append(
|
400 |
f"{format_time(time_elapsed)}<{format_time(time_remaining)}")
|
|
|
401 |
else:
|
402 |
meta_info.append(format_time(time_elapsed))
|
403 |
|
404 |
+
current_speed = Global.training_eta_predictor.get_current_speed()
|
405 |
+
if current_speed is not None:
|
406 |
+
meta_info.append(f"{current_speed:.2f}it/s")
|
407 |
+
|
408 |
+
if time_remaining >= 0:
|
409 |
+
meta_info.append(f"ETA: {format_timestamp(Global.training_eta)}")
|
410 |
+
|
411 |
params_info_html = ""
|
412 |
if Global.training_params_info_text:
|
413 |
params_info_html = f"""
|
llama_lora/utils/eta_predictor.py
CHANGED
@@ -52,3 +52,18 @@ class ETAPredictor:
|
|
52 |
print("Error predicting ETA:", e)
|
53 |
traceback.print_exc()
|
54 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
52 |
print("Error predicting ETA:", e)
|
53 |
traceback.print_exc()
|
54 |
return None
|
55 |
+
|
56 |
+
def get_current_speed(self):
|
57 |
+
if len(self.data) < 5:
|
58 |
+
return None
|
59 |
+
|
60 |
+
last = self.data[-1]
|
61 |
+
sample = self.data[-5]
|
62 |
+
if len(self.data) > 100:
|
63 |
+
sample = self.data[-2]
|
64 |
+
|
65 |
+
steps_completed = last[0] - sample[0]
|
66 |
+
time_elapsed = last[1] - sample[1]
|
67 |
+
steps_per_second = steps_completed / time_elapsed
|
68 |
+
|
69 |
+
return steps_per_second
|