Spaces:
Runtime error
Runtime error
utils package for html and tree
Browse files- app.py +83 -317
- utils/__init__.py +7 -0
- utils/html_utils.py +230 -0
- tree_utils.py → utils/tree_utils.py +5 -5
app.py
CHANGED
@@ -6,235 +6,10 @@ import numpy as np
|
|
6 |
import torch
|
7 |
from threading import Thread
|
8 |
|
9 |
-
from tree_utils import
|
|
|
10 |
PIPE = None
|
11 |
|
12 |
-
def make_script(shader_code):
|
13 |
-
# code copied and fixed(escaping single quotes to double quotes!!!) from https://webglfundamentals.org/webgl/webgl-shadertoy.html
|
14 |
-
script = ("""
|
15 |
-
<!-- Licensed under a BSD license. See license.html for license -->
|
16 |
-
<!DOCTYPE html>
|
17 |
-
<html>
|
18 |
-
<head>
|
19 |
-
<meta charset="utf-8">
|
20 |
-
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
|
21 |
-
<title>WebGL - Shadertoy</title>
|
22 |
-
<link type="text/css" href="https://webglfundamentals.org/webgl/resources/webgl-tutorials.css" rel="stylesheet" />
|
23 |
-
<style>
|
24 |
-
.divcanvas {
|
25 |
-
position: relative;
|
26 |
-
display: inline-block;
|
27 |
-
}
|
28 |
-
canvas {
|
29 |
-
display: block;
|
30 |
-
}
|
31 |
-
.playpause {
|
32 |
-
position: absolute;
|
33 |
-
left: 10px;
|
34 |
-
top: 10px;
|
35 |
-
width: 100%;
|
36 |
-
height: 100%;
|
37 |
-
font-size: 60px;
|
38 |
-
justify-content: center;
|
39 |
-
align-items: center;
|
40 |
-
color: rgba(255, 255, 255, 0.3);
|
41 |
-
transition: opacity 0.2s ease-in-out;
|
42 |
-
}
|
43 |
-
.playpausehide,
|
44 |
-
.playpause:hover {
|
45 |
-
opacity: 0;
|
46 |
-
}
|
47 |
-
.iframe .divcanvas {
|
48 |
-
display: block;
|
49 |
-
}
|
50 |
-
</style>
|
51 |
-
</head>
|
52 |
-
<body>
|
53 |
-
<div class="divcanvas">
|
54 |
-
<canvas id="canvas"></canvas>
|
55 |
-
<div class="playpause">▶</div>
|
56 |
-
</div>
|
57 |
-
\nblank canvas here indicates that some of the shadertoy specific functions are not yet supported with this implementation (like #define I believe). you can always copy and paste the code into a shadertoy.com window to try.
|
58 |
-
</body>
|
59 |
-
<!--
|
60 |
-
for most samples webgl-utils only provides shader compiling/linking and
|
61 |
-
canvas resizing because why clutter the examples with code thats the same in every sample.
|
62 |
-
See https://webglfundamentals.org/webgl/lessons/webgl-boilerplate.html
|
63 |
-
and https://webglfundamentals.org/webgl/lessons/webgl-resizing-the-canvas.html
|
64 |
-
for webgl-utils, m3, m4, and webgl-lessons-ui.
|
65 |
-
-->
|
66 |
-
<script src="https://webglfundamentals.org/webgl/resources/webgl-utils.js"></script>
|
67 |
-
<script>
|
68 |
-
"use strict";
|
69 |
-
|
70 |
-
function main() {
|
71 |
-
// Get A WebGL context
|
72 |
-
/** @type {HTMLCanvasElement} */
|
73 |
-
const canvas = document.querySelector("#canvas");
|
74 |
-
const gl = canvas.getContext("webgl");
|
75 |
-
if (!gl) {
|
76 |
-
return;
|
77 |
-
}
|
78 |
-
|
79 |
-
const vs = `
|
80 |
-
// an attribute will receive data from a buffer
|
81 |
-
attribute vec4 a_position;
|
82 |
-
|
83 |
-
// all shaders have a main function
|
84 |
-
void main() {
|
85 |
-
|
86 |
-
// gl_Position is a special variable a vertex shader
|
87 |
-
// is responsible for setting
|
88 |
-
gl_Position = a_position;
|
89 |
-
}
|
90 |
-
`;
|
91 |
-
|
92 |
-
const fs = `
|
93 |
-
precision highp float;
|
94 |
-
|
95 |
-
uniform vec2 iResolution;
|
96 |
-
uniform vec2 iMouse;
|
97 |
-
uniform float iTime;
|
98 |
-
|
99 |
-
""" + shader_code + """
|
100 |
-
void main() {
|
101 |
-
mainImage(gl_FragColor, gl_FragCoord.xy);
|
102 |
-
}
|
103 |
-
`;
|
104 |
-
|
105 |
-
// setup GLSL program
|
106 |
-
const program = webglUtils.createProgramFromSources(gl, [vs, fs]);
|
107 |
-
|
108 |
-
// look up where the vertex data needs to go.
|
109 |
-
const positionAttributeLocation = gl.getAttribLocation(program, "a_position");
|
110 |
-
|
111 |
-
// look up uniform locations
|
112 |
-
const resolutionLocation = gl.getUniformLocation(program, "iResolution");
|
113 |
-
const mouseLocation = gl.getUniformLocation(program, "iMouse");
|
114 |
-
const timeLocation = gl.getUniformLocation(program, "iTime");
|
115 |
-
|
116 |
-
// Create a buffer to put three 2d clip space points in
|
117 |
-
const positionBuffer = gl.createBuffer();
|
118 |
-
|
119 |
-
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
|
120 |
-
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
121 |
-
|
122 |
-
// fill it with a 2 triangles that cover clipspace
|
123 |
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
|
124 |
-
-1, -1, // first triangle
|
125 |
-
1, -1,
|
126 |
-
-1, 1,
|
127 |
-
-1, 1, // second triangle
|
128 |
-
1, -1,
|
129 |
-
1, 1,
|
130 |
-
]), gl.STATIC_DRAW);
|
131 |
-
|
132 |
-
const playpauseElem = document.querySelector(".playpause");
|
133 |
-
const inputElem = document.querySelector(".divcanvas");
|
134 |
-
inputElem.addEventListener("mouseover", requestFrame);
|
135 |
-
inputElem.addEventListener("mouseout", cancelFrame);
|
136 |
-
|
137 |
-
let mouseX = 0;
|
138 |
-
let mouseY = 0;
|
139 |
-
|
140 |
-
function setMousePosition(e) {
|
141 |
-
const rect = inputElem.getBoundingClientRect();
|
142 |
-
mouseX = e.clientX - rect.left;
|
143 |
-
mouseY = rect.height - (e.clientY - rect.top) - 1; // bottom is 0 in WebGL
|
144 |
-
}
|
145 |
-
|
146 |
-
inputElem.addEventListener("mousemove", setMousePosition);
|
147 |
-
inputElem.addEventListener("touchstart", (e) => {
|
148 |
-
e.preventDefault();
|
149 |
-
playpauseElem.classList.add("playpausehide");
|
150 |
-
requestFrame();
|
151 |
-
}, {passive: false});
|
152 |
-
inputElem.addEventListener("touchmove", (e) => {
|
153 |
-
e.preventDefault();
|
154 |
-
setMousePosition(e.touches[0]);
|
155 |
-
}, {passive: false});
|
156 |
-
inputElem.addEventListener("touchend", (e) => {
|
157 |
-
e.preventDefault();
|
158 |
-
playpauseElem.classList.remove("playpausehide");
|
159 |
-
cancelFrame();
|
160 |
-
}, {passive: false});
|
161 |
-
|
162 |
-
let requestId;
|
163 |
-
function requestFrame() {
|
164 |
-
if (!requestId) {
|
165 |
-
requestId = requestAnimationFrame(render);
|
166 |
-
}
|
167 |
-
}
|
168 |
-
function cancelFrame() {
|
169 |
-
if (requestId) {
|
170 |
-
cancelAnimationFrame(requestId);
|
171 |
-
requestId = undefined;
|
172 |
-
}
|
173 |
-
}
|
174 |
-
|
175 |
-
let then = 0;
|
176 |
-
let time = 0;
|
177 |
-
function render(now) {
|
178 |
-
requestId = undefined;
|
179 |
-
now *= 0.001; // convert to seconds
|
180 |
-
const elapsedTime = Math.min(now - then, 0.1);
|
181 |
-
time += elapsedTime;
|
182 |
-
then = now;
|
183 |
-
|
184 |
-
webglUtils.resizeCanvasToDisplaySize(gl.canvas);
|
185 |
-
|
186 |
-
// Tell WebGL how to convert from clip space to pixels
|
187 |
-
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
|
188 |
-
|
189 |
-
// Tell it to use our program (pair of shaders)
|
190 |
-
gl.useProgram(program);
|
191 |
-
|
192 |
-
// Turn on the attribute
|
193 |
-
gl.enableVertexAttribArray(positionAttributeLocation);
|
194 |
-
|
195 |
-
// Bind the position buffer.
|
196 |
-
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
197 |
-
|
198 |
-
// Tell the attribute how to get data out of positionBuffer (ARRAY_BUFFER)
|
199 |
-
gl.vertexAttribPointer(
|
200 |
-
positionAttributeLocation,
|
201 |
-
2, // 2 components per iteration
|
202 |
-
gl.FLOAT, // the data is 32bit floats
|
203 |
-
false, // dont normalize the data
|
204 |
-
0, // 0 = move forward size * sizeof(type) each iteration to get the next position
|
205 |
-
0, // start at the beginning of the buffer
|
206 |
-
);
|
207 |
-
|
208 |
-
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
|
209 |
-
gl.uniform2f(mouseLocation, mouseX, mouseY);
|
210 |
-
gl.uniform1f(timeLocation, time);
|
211 |
-
|
212 |
-
gl.drawArrays(
|
213 |
-
gl.TRIANGLES,
|
214 |
-
0, // offset
|
215 |
-
6, // num vertices to process
|
216 |
-
);
|
217 |
-
|
218 |
-
requestFrame();
|
219 |
-
}
|
220 |
-
|
221 |
-
requestFrame();
|
222 |
-
requestAnimationFrame(cancelFrame);
|
223 |
-
}
|
224 |
-
|
225 |
-
main();
|
226 |
-
</script>
|
227 |
-
</html>
|
228 |
-
|
229 |
-
|
230 |
-
""")
|
231 |
-
return script
|
232 |
-
|
233 |
-
def make_iframe(shader_code): #keep a single function?
|
234 |
-
script = make_script(shader_code)
|
235 |
-
return f"""<iframe width="640" height="420" srcdoc=\'{script}\' allowfullscreen></iframe>"""
|
236 |
-
|
237 |
-
|
238 |
intro_text = """
|
239 |
# Welcome to the interactive shadercoding demo.
|
240 |
This gives you access to a filtered version of the [Shadertoys](https://huggingface.co/datasets/Vipitis/Shadertoys) dataset, only shaders that consist of a single pass are available.
|
@@ -423,7 +198,7 @@ def _combine_generation_kwargs(temperature, max_new_tokens, top_p, repetition_pe
|
|
423 |
gen_kwargs["repetition_penalty"] = repetition_penalty
|
424 |
return gen_kwargs
|
425 |
|
426 |
-
def alter_body(old_code, func_id, funcs_list: list, prompt, temperature, max_new_tokens, top_p, repetition_penalty, pipeline=PIPE):
|
427 |
"""
|
428 |
Replaces the body of a function with a generated one.
|
429 |
Args:
|
@@ -457,19 +232,19 @@ def alter_body(old_code, func_id, funcs_list: list, prompt, temperature, max_new
|
|
457 |
print("no pipeline found, loading default one")
|
458 |
pipeline = _make_pipeline("Vipitis/santacoder-finetuned-Shadertoys-fine")
|
459 |
|
460 |
-
func_start_idx =
|
461 |
identifier_str = func_node.child_by_field_name("type").text.decode() + " " + func_node.child_by_field_name("declarator").text.decode() #func_start_idx:body_start_idx?
|
462 |
body_node = func_node.child_by_field_name("body")
|
463 |
-
body_start_idx =
|
464 |
-
body_end_idx =
|
465 |
print(f"{old_code[body_start_idx:body_end_idx]=}")
|
466 |
model_context = identifier_str # base case
|
467 |
# add any comments at the beginning of the function to the model_context
|
468 |
# second_child = func_node.child_by_field_name("body").children[1] #might error out?
|
469 |
-
docstring =
|
470 |
if docstring:
|
471 |
model_context = model_context + "\n" + docstring
|
472 |
-
model_context =
|
473 |
if prompt != "":
|
474 |
model_context = f"//avialable functions: {','.join([n.child_by_field_name('declarator').text.decode() for n in funcs_list])}\n" + model_context #prepend available functions
|
475 |
model_context = "//Title: " + prompt + "\n" + model_context #prepend user prompt/title
|
@@ -486,7 +261,7 @@ def alter_body(old_code, func_id, funcs_list: list, prompt, temperature, max_new
|
|
486 |
print(f"{ctx_with_generation=}")
|
487 |
try:
|
488 |
#strip the body
|
489 |
-
first_gened_func =
|
490 |
except IndexError:
|
491 |
print("generation wasn't a full function.")
|
492 |
altered_code = old_code[:func_start_idx] + model_context + generation + "//the generation didn't complete the function!\n" + old_code[body_end_idx:] #needs a newline to break out of the comment.
|
@@ -500,13 +275,8 @@ def alter_body(old_code, func_id, funcs_list: list, prompt, temperature, max_new
|
|
500 |
yield altered_code, pipeline #yield once so it updates? -> works... gg but doesn't seem to do it for the dropdown
|
501 |
return altered_code, pipeline #never gets used by the code block? maybe I need to yield it first? but works in the ov_notebook
|
502 |
|
503 |
-
def add_history(func_id, orig_rtn, gened_rtn, history):
|
504 |
-
# is this a list? or a JSON dict?
|
505 |
-
history[func_id] = (orig_rtn, gened_rtn)
|
506 |
-
return history, history
|
507 |
-
|
508 |
def list_dropdown(in_code): #only used for auto update, not on sample pick?
|
509 |
-
funcs =
|
510 |
|
511 |
# print(f"updating drop down to:{func_identifiers=}")
|
512 |
func_identifiers = [f"{idx:2d}: {n.child_by_field_name('declarator').text.decode()}" for idx, n in enumerate(funcs)]
|
@@ -514,83 +284,6 @@ def list_dropdown(in_code): #only used for auto update, not on sample pick?
|
|
514 |
print(f"updating drop down to:{func_identifiers}")
|
515 |
return funcs, gr.Dropdown.update(choices=func_identifiers)
|
516 |
|
517 |
-
def construct_embed(source_url):
|
518 |
-
shader_id = source_url.split("/")[-1]
|
519 |
-
return f'<iframe width="640" height="360" frameborder="0" src="https://www.shadertoy.com/embed/{shader_id}?gui=true&t=0&paused=true&muted=true" allowfullscreen></iframe>'
|
520 |
-
|
521 |
-
with gr.Blocks() as site:
|
522 |
-
top_md = gr.Markdown(intro_text)
|
523 |
-
model_cp = gr.Textbox(value="Vipitis/santacoder-finetuned-Shadertoys-fine", label="Model Checkpoint (Enter to load!)", interactive=True)
|
524 |
-
sample_idx = gr.Slider(minimum=0, maximum=10513, value=3211, label="pick sample from dataset", step=1.0)
|
525 |
-
func_dropdown = gr.Dropdown(value=["0: edit the Code (or load a shader) to update this dropdown"], label="chose a function to modify") #breaks if I add a string in before that? #TODO: use type="index" to get int - always gives None?
|
526 |
-
prompt_text = gr.Textbox(value="the title used by the model has generation hint", label="prompt text", info="leave blank to skip", interactive=True)
|
527 |
-
with gr.Accordion("Advanced settings", open=False): # from: https://huggingface.co/spaces/bigcode/bigcode-playground/blob/main/app.py
|
528 |
-
with gr.Row():
|
529 |
-
column_1, column_2 = gr.Column(), gr.Column()
|
530 |
-
with column_1:
|
531 |
-
temperature = gr.Slider(
|
532 |
-
label="Temperature",
|
533 |
-
value=0.2, #start out at 0 to do greedy? or will there be an error?
|
534 |
-
minimum=0.0,
|
535 |
-
maximum=1.0,
|
536 |
-
step=0.05,
|
537 |
-
interactive=True,
|
538 |
-
info="Higher values produce more diverse outputs",
|
539 |
-
)
|
540 |
-
max_new_tokens = gr.Slider(
|
541 |
-
label="Max new tokens",
|
542 |
-
value=265,
|
543 |
-
minimum=0,
|
544 |
-
maximum=2048, #this could be inferred from the model?
|
545 |
-
step=32,
|
546 |
-
interactive=True,
|
547 |
-
info="The maximum numbers of new tokens",
|
548 |
-
)
|
549 |
-
with column_2:
|
550 |
-
top_p = gr.Slider(
|
551 |
-
label="Top-p (nucleus sampling)",
|
552 |
-
value=0.90,
|
553 |
-
minimum=0.0,
|
554 |
-
maximum=1,
|
555 |
-
step=0.05,
|
556 |
-
interactive=True,
|
557 |
-
info="Higher values sample more low-probability tokens",
|
558 |
-
)
|
559 |
-
repetition_penalty = gr.Slider(
|
560 |
-
label="Repetition penalty",
|
561 |
-
value=1.2,
|
562 |
-
minimum=1.0,
|
563 |
-
maximum=2.0,
|
564 |
-
step=0.05,
|
565 |
-
interactive=True,
|
566 |
-
info="Penalize repeated tokens",
|
567 |
-
)
|
568 |
-
with gr.Row():
|
569 |
-
gen_return_button = gr.Button("generate a alternate return statement", label="generate return", scale=0)
|
570 |
-
gen_func_button = gr.Button("generate an alternate function body", label="generate function", scale=1)
|
571 |
-
with gr.Row():
|
572 |
-
with gr.Column():
|
573 |
-
source_embed = gr.HTML('<iframe width="640" height="360" frameborder="0" src="" allowfullscreen></iframe>', label="How this shader originally renders")
|
574 |
-
our_embed = gr.HTML(label="glsl render of the current code")
|
575 |
-
sample_code = gr.Code(new_shadertoy_code, label="Current Code (will update changes you generate)", language=None)
|
576 |
-
bot_md = gr.Markdown(outro_text)
|
577 |
-
sample_pass = gr.State(value={})
|
578 |
-
pipe = gr.State(value=PIPE)
|
579 |
-
pipe.value=_make_pipeline("Vipitis/santacoder-finetuned-Shadertoys-fine") # set a default like this?
|
580 |
-
funcs = gr.State(value=[])
|
581 |
-
# funcs.value.append(list_dropdown(sample_code.value)[0]) #to circumvent the json issue?
|
582 |
-
# hist_state = gr.State(Value={})
|
583 |
-
# history_table = gr.JSON()
|
584 |
-
|
585 |
-
model_cp.submit(fn=_make_pipeline, inputs=[model_cp], outputs=[pipe]) # how can we trigger this on load?
|
586 |
-
sample_idx.release(fn=grab_sample, inputs=[sample_idx], outputs=[sample_pass, sample_code, prompt_text, source_embed]) #funcs here?
|
587 |
-
gen_return_button.click(fn=alter_return, inputs=[sample_code, func_dropdown, temperature, max_new_tokens, top_p, repetition_penalty, pipe], outputs=[sample_code])
|
588 |
-
gen_func_button.click(fn=alter_body, inputs=[sample_code, func_dropdown, funcs, prompt_text, temperature, max_new_tokens, top_p, repetition_penalty, pipe], outputs=[sample_code, pipe]).then(
|
589 |
-
fn=list_dropdown, inputs=[sample_code], outputs=[funcs, func_dropdown]
|
590 |
-
)
|
591 |
-
sample_code.change(fn=list_dropdown, inputs=[sample_code], outputs=[funcs, func_dropdown]).then(
|
592 |
-
fn=make_iframe, inputs=[sample_code], outputs=[our_embed])
|
593 |
-
|
594 |
if __name__ == "__main__": #works on huggingface?
|
595 |
passes_dataset = datasets.load_dataset("Vipitis/Shadertoys")
|
596 |
single_passes = passes_dataset.filter(lambda x: not x["has_inputs"] and x["num_passes"] == 1) #could also include shaders with no extra functions.
|
@@ -598,5 +291,78 @@ if __name__ == "__main__": #works on huggingface?
|
|
598 |
all_single_passes = datasets.concatenate_datasets([single_passes["train"], single_passes["test"]])
|
599 |
num_samples = len(all_single_passes)
|
600 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
601 |
site.queue()
|
602 |
site.launch()
|
|
|
6 |
import torch
|
7 |
from threading import Thread
|
8 |
|
9 |
+
from utils.tree_utils import parse_functions, get_docstrings, grab_before_comments, line_chr2char
|
10 |
+
from utils.html_utils import make_iframe, construct_embed
|
11 |
PIPE = None
|
12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13 |
intro_text = """
|
14 |
# Welcome to the interactive shadercoding demo.
|
15 |
This gives you access to a filtered version of the [Shadertoys](https://huggingface.co/datasets/Vipitis/Shadertoys) dataset, only shaders that consist of a single pass are available.
|
|
|
198 |
gen_kwargs["repetition_penalty"] = repetition_penalty
|
199 |
return gen_kwargs
|
200 |
|
201 |
+
def alter_body(old_code, func_id, funcs_list: list, prompt="", temperature=0.2, max_new_tokens=512, top_p=.95, repetition_penalty=1.2, pipeline=PIPE):
|
202 |
"""
|
203 |
Replaces the body of a function with a generated one.
|
204 |
Args:
|
|
|
232 |
print("no pipeline found, loading default one")
|
233 |
pipeline = _make_pipeline("Vipitis/santacoder-finetuned-Shadertoys-fine")
|
234 |
|
235 |
+
func_start_idx = line_chr2char(old_code, func_node.start_point[0], func_node.start_point[1])
|
236 |
identifier_str = func_node.child_by_field_name("type").text.decode() + " " + func_node.child_by_field_name("declarator").text.decode() #func_start_idx:body_start_idx?
|
237 |
body_node = func_node.child_by_field_name("body")
|
238 |
+
body_start_idx = line_chr2char(old_code, body_node.start_point[0], body_node.start_point[1])
|
239 |
+
body_end_idx = line_chr2char(old_code, body_node.end_point[0], body_node.end_point[1])
|
240 |
print(f"{old_code[body_start_idx:body_end_idx]=}")
|
241 |
model_context = identifier_str # base case
|
242 |
# add any comments at the beginning of the function to the model_context
|
243 |
# second_child = func_node.child_by_field_name("body").children[1] #might error out?
|
244 |
+
docstring = get_docstrings(func_node) #might be empty?
|
245 |
if docstring:
|
246 |
model_context = model_context + "\n" + docstring
|
247 |
+
model_context = grab_before_comments(func_node) + model_context #prepend comments
|
248 |
if prompt != "":
|
249 |
model_context = f"//avialable functions: {','.join([n.child_by_field_name('declarator').text.decode() for n in funcs_list])}\n" + model_context #prepend available functions
|
250 |
model_context = "//Title: " + prompt + "\n" + model_context #prepend user prompt/title
|
|
|
261 |
print(f"{ctx_with_generation=}")
|
262 |
try:
|
263 |
#strip the body
|
264 |
+
first_gened_func = parse_functions(ctx_with_generation)[0] # truncate generation to a single function?
|
265 |
except IndexError:
|
266 |
print("generation wasn't a full function.")
|
267 |
altered_code = old_code[:func_start_idx] + model_context + generation + "//the generation didn't complete the function!\n" + old_code[body_end_idx:] #needs a newline to break out of the comment.
|
|
|
275 |
yield altered_code, pipeline #yield once so it updates? -> works... gg but doesn't seem to do it for the dropdown
|
276 |
return altered_code, pipeline #never gets used by the code block? maybe I need to yield it first? but works in the ov_notebook
|
277 |
|
|
|
|
|
|
|
|
|
|
|
278 |
def list_dropdown(in_code): #only used for auto update, not on sample pick?
|
279 |
+
funcs = parse_functions(in_code)
|
280 |
|
281 |
# print(f"updating drop down to:{func_identifiers=}")
|
282 |
func_identifiers = [f"{idx:2d}: {n.child_by_field_name('declarator').text.decode()}" for idx, n in enumerate(funcs)]
|
|
|
284 |
print(f"updating drop down to:{func_identifiers}")
|
285 |
return funcs, gr.Dropdown.update(choices=func_identifiers)
|
286 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
287 |
if __name__ == "__main__": #works on huggingface?
|
288 |
passes_dataset = datasets.load_dataset("Vipitis/Shadertoys")
|
289 |
single_passes = passes_dataset.filter(lambda x: not x["has_inputs"] and x["num_passes"] == 1) #could also include shaders with no extra functions.
|
|
|
291 |
all_single_passes = datasets.concatenate_datasets([single_passes["train"], single_passes["test"]])
|
292 |
num_samples = len(all_single_passes)
|
293 |
|
294 |
+
with gr.Blocks() as site:
|
295 |
+
top_md = gr.Markdown(intro_text)
|
296 |
+
model_cp = gr.Textbox(value="Vipitis/santacoder-finetuned-Shadertoys-fine", label="Model Checkpoint (Enter to load!)", interactive=True)
|
297 |
+
sample_idx = gr.Slider(minimum=0, maximum=10513, value=3211, label="pick sample from dataset", step=1.0)
|
298 |
+
func_dropdown = gr.Dropdown(value=["0: edit the Code (or load a shader) to update this dropdown"], label="chose a function to modify") #breaks if I add a string in before that? #TODO: use type="index" to get int - always gives None?
|
299 |
+
prompt_text = gr.Textbox(value="the title used by the model has generation hint", label="prompt text", info="leave blank to skip", interactive=True)
|
300 |
+
with gr.Accordion("Advanced settings", open=False): # from: https://huggingface.co/spaces/bigcode/bigcode-playground/blob/main/app.py
|
301 |
+
with gr.Row():
|
302 |
+
column_1, column_2 = gr.Column(), gr.Column()
|
303 |
+
with column_1:
|
304 |
+
temperature = gr.Slider(
|
305 |
+
label="Temperature",
|
306 |
+
value=0.2, #start out at 0 to do greedy? or will there be an error?
|
307 |
+
minimum=0.0,
|
308 |
+
maximum=1.0,
|
309 |
+
step=0.05,
|
310 |
+
interactive=True,
|
311 |
+
info="Higher values produce more diverse outputs",
|
312 |
+
)
|
313 |
+
max_new_tokens = gr.Slider(
|
314 |
+
label="Max new tokens",
|
315 |
+
value=265,
|
316 |
+
minimum=0,
|
317 |
+
maximum=2048, #this could be inferred from the model?
|
318 |
+
step=32,
|
319 |
+
interactive=True,
|
320 |
+
info="The maximum numbers of new tokens",
|
321 |
+
)
|
322 |
+
with column_2:
|
323 |
+
top_p = gr.Slider(
|
324 |
+
label="Top-p (nucleus sampling)",
|
325 |
+
value=0.90,
|
326 |
+
minimum=0.0,
|
327 |
+
maximum=1,
|
328 |
+
step=0.05,
|
329 |
+
interactive=True,
|
330 |
+
info="Higher values sample more low-probability tokens",
|
331 |
+
)
|
332 |
+
repetition_penalty = gr.Slider(
|
333 |
+
label="Repetition penalty",
|
334 |
+
value=1.2,
|
335 |
+
minimum=1.0,
|
336 |
+
maximum=2.0,
|
337 |
+
step=0.05,
|
338 |
+
interactive=True,
|
339 |
+
info="Penalize repeated tokens",
|
340 |
+
)
|
341 |
+
with gr.Row():
|
342 |
+
gen_return_button = gr.Button("generate a alternate return statement", label="generate return", scale=0)
|
343 |
+
gen_func_button = gr.Button("generate an alternate function body", label="generate function", scale=1)
|
344 |
+
with gr.Row():
|
345 |
+
with gr.Column():
|
346 |
+
source_embed = gr.HTML('<iframe width="640" height="360" frameborder="0" src="" allowfullscreen></iframe>', label="How this shader originally renders")
|
347 |
+
our_embed = gr.HTML(label="glsl render of the current code")
|
348 |
+
sample_code = gr.Code(new_shadertoy_code, label="Current Code (will update changes you generate)", language=None)
|
349 |
+
bot_md = gr.Markdown(outro_text)
|
350 |
+
sample_pass = gr.State(value={})
|
351 |
+
pipe = gr.State(value=PIPE)
|
352 |
+
pipe.value=_make_pipeline("Vipitis/santacoder-finetuned-Shadertoys-fine") # set a default like this?
|
353 |
+
funcs = gr.State(value=[])
|
354 |
+
# funcs.value.append(list_dropdown(sample_code.value)[0]) #to circumvent the json issue?
|
355 |
+
# hist_state = gr.State(Value={})
|
356 |
+
# history_table = gr.JSON()
|
357 |
+
|
358 |
+
model_cp.submit(fn=_make_pipeline, inputs=[model_cp], outputs=[pipe]) # how can we trigger this on load?
|
359 |
+
sample_idx.release(fn=grab_sample, inputs=[sample_idx], outputs=[sample_pass, sample_code, prompt_text, source_embed]) #funcs here?
|
360 |
+
gen_return_button.click(fn=alter_return, inputs=[sample_code, func_dropdown, temperature, max_new_tokens, top_p, repetition_penalty, pipe], outputs=[sample_code])
|
361 |
+
gen_func_button.click(fn=alter_body, inputs=[sample_code, func_dropdown, funcs, prompt_text, temperature, max_new_tokens, top_p, repetition_penalty, pipe], outputs=[sample_code, pipe]).then(
|
362 |
+
fn=list_dropdown, inputs=[sample_code], outputs=[funcs, func_dropdown]
|
363 |
+
)
|
364 |
+
sample_code.change(fn=list_dropdown, inputs=[sample_code], outputs=[funcs, func_dropdown]).then(
|
365 |
+
fn=make_iframe, inputs=[sample_code], outputs=[our_embed])
|
366 |
+
|
367 |
site.queue()
|
368 |
site.launch()
|
utils/__init__.py
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from .tree_utils import (parse_functions, get_docstrings, grab_before_comments, line_chr2char)
|
2 |
+
from .html_utils import (make_iframe, make_script, construct_embed)
|
3 |
+
|
4 |
+
tree_funcs = ["parse_functions", "get_docstrings", "grab_before_comments", "line_chr2char"]
|
5 |
+
html_funcs = ["make_iframe", "make_script", "construct_embed"]
|
6 |
+
|
7 |
+
__all__ = tree_funcs + html_funcs
|
utils/html_utils.py
ADDED
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
def construct_embed(source_url):
|
3 |
+
shader_id = source_url.split("/")[-1]
|
4 |
+
return f'<iframe width="640" height="360" frameborder="0" src="https://www.shadertoy.com/embed/{shader_id}?gui=true&t=0&paused=true&muted=true" allowfullscreen></iframe>'
|
5 |
+
|
6 |
+
def make_iframe(shader_code): #keep a single function?
|
7 |
+
script = make_script(shader_code)
|
8 |
+
return f"""<iframe width="640" height="420" srcdoc=\'{script}\' allowfullscreen></iframe>"""
|
9 |
+
|
10 |
+
|
11 |
+
def make_script(shader_code):
|
12 |
+
# code copied and fixed(escaping single quotes to double quotes!!!) from https://webglfundamentals.org/webgl/webgl-shadertoy.html
|
13 |
+
script = ("""
|
14 |
+
<!-- Licensed under a BSD license. See license.html for license -->
|
15 |
+
<!DOCTYPE html>
|
16 |
+
<html>
|
17 |
+
<head>
|
18 |
+
<meta charset="utf-8">
|
19 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
|
20 |
+
<title>WebGL - Shadertoy</title>
|
21 |
+
<link type="text/css" href="https://webglfundamentals.org/webgl/resources/webgl-tutorials.css" rel="stylesheet" />
|
22 |
+
<style>
|
23 |
+
.divcanvas {
|
24 |
+
position: relative;
|
25 |
+
display: inline-block;
|
26 |
+
}
|
27 |
+
canvas {
|
28 |
+
display: block;
|
29 |
+
}
|
30 |
+
.playpause {
|
31 |
+
position: absolute;
|
32 |
+
left: 10px;
|
33 |
+
top: 10px;
|
34 |
+
width: 100%;
|
35 |
+
height: 100%;
|
36 |
+
font-size: 60px;
|
37 |
+
justify-content: center;
|
38 |
+
align-items: center;
|
39 |
+
color: rgba(255, 255, 255, 0.3);
|
40 |
+
transition: opacity 0.2s ease-in-out;
|
41 |
+
}
|
42 |
+
.playpausehide,
|
43 |
+
.playpause:hover {
|
44 |
+
opacity: 0;
|
45 |
+
}
|
46 |
+
.iframe .divcanvas {
|
47 |
+
display: block;
|
48 |
+
}
|
49 |
+
</style>
|
50 |
+
</head>
|
51 |
+
<body>
|
52 |
+
<div class="divcanvas">
|
53 |
+
<canvas id="canvas"></canvas>
|
54 |
+
<div class="playpause">▶</div>
|
55 |
+
</div>
|
56 |
+
\nblank canvas here indicates that some of the shadertoy specific functions are not yet supported with this implementation (like #define I believe). you can always copy and paste the code into a shadertoy.com window to try.
|
57 |
+
</body>
|
58 |
+
<!--
|
59 |
+
for most samples webgl-utils only provides shader compiling/linking and
|
60 |
+
canvas resizing because why clutter the examples with code thats the same in every sample.
|
61 |
+
See https://webglfundamentals.org/webgl/lessons/webgl-boilerplate.html
|
62 |
+
and https://webglfundamentals.org/webgl/lessons/webgl-resizing-the-canvas.html
|
63 |
+
for webgl-utils, m3, m4, and webgl-lessons-ui.
|
64 |
+
-->
|
65 |
+
<script src="https://webglfundamentals.org/webgl/resources/webgl-utils.js"></script>
|
66 |
+
<script>
|
67 |
+
"use strict";
|
68 |
+
|
69 |
+
function main() {
|
70 |
+
// Get A WebGL context
|
71 |
+
/** @type {HTMLCanvasElement} */
|
72 |
+
const canvas = document.querySelector("#canvas");
|
73 |
+
const gl = canvas.getContext("webgl");
|
74 |
+
if (!gl) {
|
75 |
+
return;
|
76 |
+
}
|
77 |
+
|
78 |
+
const vs = `
|
79 |
+
// an attribute will receive data from a buffer
|
80 |
+
attribute vec4 a_position;
|
81 |
+
|
82 |
+
// all shaders have a main function
|
83 |
+
void main() {
|
84 |
+
|
85 |
+
// gl_Position is a special variable a vertex shader
|
86 |
+
// is responsible for setting
|
87 |
+
gl_Position = a_position;
|
88 |
+
}
|
89 |
+
`;
|
90 |
+
|
91 |
+
const fs = `
|
92 |
+
precision highp float;
|
93 |
+
|
94 |
+
uniform vec2 iResolution;
|
95 |
+
uniform vec2 iMouse;
|
96 |
+
uniform float iTime;
|
97 |
+
|
98 |
+
""" + shader_code + """
|
99 |
+
void main() {
|
100 |
+
mainImage(gl_FragColor, gl_FragCoord.xy);
|
101 |
+
}
|
102 |
+
`;
|
103 |
+
|
104 |
+
// setup GLSL program
|
105 |
+
const program = webglUtils.createProgramFromSources(gl, [vs, fs]);
|
106 |
+
|
107 |
+
// look up where the vertex data needs to go.
|
108 |
+
const positionAttributeLocation = gl.getAttribLocation(program, "a_position");
|
109 |
+
|
110 |
+
// look up uniform locations
|
111 |
+
const resolutionLocation = gl.getUniformLocation(program, "iResolution");
|
112 |
+
const mouseLocation = gl.getUniformLocation(program, "iMouse");
|
113 |
+
const timeLocation = gl.getUniformLocation(program, "iTime");
|
114 |
+
|
115 |
+
// Create a buffer to put three 2d clip space points in
|
116 |
+
const positionBuffer = gl.createBuffer();
|
117 |
+
|
118 |
+
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
|
119 |
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
120 |
+
|
121 |
+
// fill it with a 2 triangles that cover clipspace
|
122 |
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
|
123 |
+
-1, -1, // first triangle
|
124 |
+
1, -1,
|
125 |
+
-1, 1,
|
126 |
+
-1, 1, // second triangle
|
127 |
+
1, -1,
|
128 |
+
1, 1,
|
129 |
+
]), gl.STATIC_DRAW);
|
130 |
+
|
131 |
+
const playpauseElem = document.querySelector(".playpause");
|
132 |
+
const inputElem = document.querySelector(".divcanvas");
|
133 |
+
inputElem.addEventListener("mouseover", requestFrame);
|
134 |
+
inputElem.addEventListener("mouseout", cancelFrame);
|
135 |
+
|
136 |
+
let mouseX = 0;
|
137 |
+
let mouseY = 0;
|
138 |
+
|
139 |
+
function setMousePosition(e) {
|
140 |
+
const rect = inputElem.getBoundingClientRect();
|
141 |
+
mouseX = e.clientX - rect.left;
|
142 |
+
mouseY = rect.height - (e.clientY - rect.top) - 1; // bottom is 0 in WebGL
|
143 |
+
}
|
144 |
+
|
145 |
+
inputElem.addEventListener("mousemove", setMousePosition);
|
146 |
+
inputElem.addEventListener("touchstart", (e) => {
|
147 |
+
e.preventDefault();
|
148 |
+
playpauseElem.classList.add("playpausehide");
|
149 |
+
requestFrame();
|
150 |
+
}, {passive: false});
|
151 |
+
inputElem.addEventListener("touchmove", (e) => {
|
152 |
+
e.preventDefault();
|
153 |
+
setMousePosition(e.touches[0]);
|
154 |
+
}, {passive: false});
|
155 |
+
inputElem.addEventListener("touchend", (e) => {
|
156 |
+
e.preventDefault();
|
157 |
+
playpauseElem.classList.remove("playpausehide");
|
158 |
+
cancelFrame();
|
159 |
+
}, {passive: false});
|
160 |
+
|
161 |
+
let requestId;
|
162 |
+
function requestFrame() {
|
163 |
+
if (!requestId) {
|
164 |
+
requestId = requestAnimationFrame(render);
|
165 |
+
}
|
166 |
+
}
|
167 |
+
function cancelFrame() {
|
168 |
+
if (requestId) {
|
169 |
+
cancelAnimationFrame(requestId);
|
170 |
+
requestId = undefined;
|
171 |
+
}
|
172 |
+
}
|
173 |
+
|
174 |
+
let then = 0;
|
175 |
+
let time = 0;
|
176 |
+
function render(now) {
|
177 |
+
requestId = undefined;
|
178 |
+
now *= 0.001; // convert to seconds
|
179 |
+
const elapsedTime = Math.min(now - then, 0.1);
|
180 |
+
time += elapsedTime;
|
181 |
+
then = now;
|
182 |
+
|
183 |
+
webglUtils.resizeCanvasToDisplaySize(gl.canvas);
|
184 |
+
|
185 |
+
// Tell WebGL how to convert from clip space to pixels
|
186 |
+
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
|
187 |
+
|
188 |
+
// Tell it to use our program (pair of shaders)
|
189 |
+
gl.useProgram(program);
|
190 |
+
|
191 |
+
// Turn on the attribute
|
192 |
+
gl.enableVertexAttribArray(positionAttributeLocation);
|
193 |
+
|
194 |
+
// Bind the position buffer.
|
195 |
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
196 |
+
|
197 |
+
// Tell the attribute how to get data out of positionBuffer (ARRAY_BUFFER)
|
198 |
+
gl.vertexAttribPointer(
|
199 |
+
positionAttributeLocation,
|
200 |
+
2, // 2 components per iteration
|
201 |
+
gl.FLOAT, // the data is 32bit floats
|
202 |
+
false, // dont normalize the data
|
203 |
+
0, // 0 = move forward size * sizeof(type) each iteration to get the next position
|
204 |
+
0, // start at the beginning of the buffer
|
205 |
+
);
|
206 |
+
|
207 |
+
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
|
208 |
+
gl.uniform2f(mouseLocation, mouseX, mouseY);
|
209 |
+
gl.uniform1f(timeLocation, time);
|
210 |
+
|
211 |
+
gl.drawArrays(
|
212 |
+
gl.TRIANGLES,
|
213 |
+
0, // offset
|
214 |
+
6, // num vertices to process
|
215 |
+
);
|
216 |
+
|
217 |
+
requestFrame();
|
218 |
+
}
|
219 |
+
|
220 |
+
requestFrame();
|
221 |
+
requestAnimationFrame(cancelFrame);
|
222 |
+
}
|
223 |
+
|
224 |
+
main();
|
225 |
+
</script>
|
226 |
+
</html>
|
227 |
+
|
228 |
+
|
229 |
+
""")
|
230 |
+
return script
|
tree_utils.py → utils/tree_utils.py
RENAMED
@@ -1,13 +1,13 @@
|
|
1 |
import tree_sitter
|
2 |
from tree_sitter import Language, Parser
|
3 |
|
4 |
-
Language.build_library("./build/my-languages.so", ['tree-sitter-glsl'])
|
5 |
GLSL_LANGUAGE = Language('./build/my-languages.so', 'glsl')
|
6 |
parser = Parser()
|
7 |
parser.set_language(GLSL_LANGUAGE)
|
8 |
|
9 |
|
10 |
-
def
|
11 |
"""
|
12 |
returns all functions in the code as their actual nodes.
|
13 |
includes any comment made directly after the function definition or diretly after #copilot trigger
|
@@ -18,7 +18,7 @@ def _parse_functions(in_code):
|
|
18 |
return funcs
|
19 |
|
20 |
|
21 |
-
def
|
22 |
"""
|
23 |
returns the docstring of a function node
|
24 |
"""
|
@@ -31,7 +31,7 @@ def _get_docstrings(func_node):
|
|
31 |
return docstring
|
32 |
|
33 |
|
34 |
-
def
|
35 |
"""
|
36 |
returns the comments that happen just before a function node
|
37 |
"""
|
@@ -47,7 +47,7 @@ def _grab_before_comments(func_node):
|
|
47 |
return precomment
|
48 |
return precomment
|
49 |
|
50 |
-
def
|
51 |
"""
|
52 |
returns the character index at the given line and character index.
|
53 |
"""
|
|
|
1 |
import tree_sitter
|
2 |
from tree_sitter import Language, Parser
|
3 |
|
4 |
+
Language.build_library("./build/my-languages.so", ['../tree-sitter-glsl'])
|
5 |
GLSL_LANGUAGE = Language('./build/my-languages.so', 'glsl')
|
6 |
parser = Parser()
|
7 |
parser.set_language(GLSL_LANGUAGE)
|
8 |
|
9 |
|
10 |
+
def parse_functions(in_code):
|
11 |
"""
|
12 |
returns all functions in the code as their actual nodes.
|
13 |
includes any comment made directly after the function definition or diretly after #copilot trigger
|
|
|
18 |
return funcs
|
19 |
|
20 |
|
21 |
+
def get_docstrings(func_node):
|
22 |
"""
|
23 |
returns the docstring of a function node
|
24 |
"""
|
|
|
31 |
return docstring
|
32 |
|
33 |
|
34 |
+
def grab_before_comments(func_node):
|
35 |
"""
|
36 |
returns the comments that happen just before a function node
|
37 |
"""
|
|
|
47 |
return precomment
|
48 |
return precomment
|
49 |
|
50 |
+
def line_chr2char(text, line_idx, chr_idx):
|
51 |
"""
|
52 |
returns the character index at the given line and character index.
|
53 |
"""
|