checkpoint

viewer/src/lib/splat.js/cameras/Camera.ts

@@ -1,14 +1,18 @@
 import { Object3D } from "../core/Object3D";
 import { Matrix4 } from "../math/Matrix4";
+import { Vector3 } from "../math/Vector3";
 
 class Camera extends Object3D {
     projectionMatrix: Matrix4;
 
-    constructor() {
+    constructor(position: Vector3 = new Vector3(0, 0, -5)) {
         super();
 
+        this.position = position;
         this.projectionMatrix = new Matrix4();
     }
+
+    updateProjectionMatrix(width: number, height: number): void {}
 }
 
 export { Camera };

viewer/src/lib/splat.js/core/Scene.ts

@@ -2,16 +2,19 @@ import { Object3D } from "./Object3D";
 
 class Scene extends Object3D {
     data: Uint8Array;
+    vertexCount: number;
 
     constructor() {
         super();
 
         this.data = new Uint8Array(0);
+        this.vertexCount = 0;
     }
 
-    setData(data: Uint8Array): void {
+    setData(data: Uint8Array, vertexCount: number): void {
         this.data = data;
-        console.log(data);
+        this.vertexCount = vertexCount;
+        console.log("setData", vertexCount);
     }
 }
 

viewer/src/lib/splat.js/index.ts

@@ -1,4 +1,5 @@
 export { Camera } from "./cameras/Camera";
+export { PerspectiveCamera } from "./cameras/PerspectiveCamera";
 export { Renderer } from "./renderers/Renderer";
 export { Scene } from "./core/Scene";
 

viewer/src/lib/splat.js/loaders/Loader.ts

@@ -27,7 +27,10 @@ class Loader {
             onProgress?.(bytesRead / contentLength);
         }
 
-        scene.setData(data);
+        const rowLength = 3 * 4 + 3 * 4 + 4 + 4;
+        const vertexCount = data.length / rowLength;
+
+        scene.setData(data, vertexCount);
     }
 }
 

viewer/src/lib/splat.js/math/Matrix3.ts

@@ -0,0 +1,31 @@
+class Matrix3 {
+    buffer: number[];
+
+    // prettier-ignore
+    constructor(n11: number = 1, n12: number = 0, n13: number = 0,
+        n21: number = 0, n22: number = 1, n23: number = 0,
+        n31: number = 0, n32: number = 0, n33: number = 1) {
+        this.buffer = new Array(9);
+
+        this.set(
+            n11, n12, n13,
+            n21, n22, n23,
+            n31, n32, n33
+        );
+    }
+
+    // prettier-ignore
+    set(n11: number, n12: number, n13: number,
+        n21: number, n22: number, n23: number,
+        n31: number, n32: number, n33: number): Matrix3 {
+        const e = this.buffer;
+
+        e[0] = n11; e[1] = n12; e[2] = n13;
+        e[3] = n21; e[4] = n22; e[5] = n23;
+        e[6] = n31; e[7] = n32; e[8] = n33;
+
+        return this;
+    }
+}
+
+export { Matrix3 };

viewer/src/lib/splat.js/math/Matrix4.ts

@@ -1,12 +1,12 @@
 class Matrix4 {
-    elements: number[];
+    buffer: number[];
 
     // prettier-ignore
     constructor(n11: number = 1, n12: number = 0, n13: number = 0, n14: number = 0, 
                 n21: number = 0, n22: number = 1, n23: number = 0, n24: number = 0, 
                 n31: number = 0, n32: number = 0, n33: number = 1, n34: number = 0, 
                 n41: number = 0, n42: number = 0, n43: number = 0, n44: number = 1) {
-        this.elements = new Array(16);
+        this.buffer = new Array(16);
 
         this.set(
             n11, n12, n13, n14, 
@@ -21,15 +21,38 @@ class Matrix4 {
         n21: number, n22: number, n23: number, n24: number,
         n31: number, n32: number, n33: number, n34: number,
         n41: number, n42: number, n43: number, n44: number) : Matrix4 {
-        const e = this.elements;
+        const e = this.buffer;
 
-        e[0] = n11; e[4] = n12; e[8] = n13; e[12] = n14;
-        e[1] = n21; e[5] = n22; e[9] = n23; e[13] = n24;
-        e[2] = n31; e[6] = n32; e[10] = n33; e[14] = n34;
-        e[3] = n41; e[7] = n42; e[11] = n43; e[15] = n44;
+        e[0] = n11; e[1] = n12; e[2] = n13; e[3] = n14;
+        e[4] = n21; e[5] = n22; e[6] = n23; e[7] = n24;
+        e[8] = n31; e[9] = n32; e[10] = n33; e[11] = n34;
+        e[12] = n41; e[13] = n42; e[14] = n43; e[15] = n44;
 
         return this;
     }
+
+    multiply(m: Matrix4): Matrix4 {
+        const a = this.buffer;
+        const b = m.buffer;
+        return new Matrix4(
+            b[0] * a[0] + b[1] * a[4] + b[2] * a[8] + b[3] * a[12],
+            b[0] * a[1] + b[1] * a[5] + b[2] * a[9] + b[3] * a[13],
+            b[0] * a[2] + b[1] * a[6] + b[2] * a[10] + b[3] * a[14],
+            b[0] * a[3] + b[1] * a[7] + b[2] * a[11] + b[3] * a[15],
+            b[4] * a[0] + b[5] * a[4] + b[6] * a[8] + b[7] * a[12],
+            b[4] * a[1] + b[5] * a[5] + b[6] * a[9] + b[7] * a[13],
+            b[4] * a[2] + b[5] * a[6] + b[6] * a[10] + b[7] * a[14],
+            b[4] * a[3] + b[5] * a[7] + b[6] * a[11] + b[7] * a[15],
+            b[8] * a[0] + b[9] * a[4] + b[10] * a[8] + b[11] * a[12],
+            b[8] * a[1] + b[9] * a[5] + b[10] * a[9] + b[11] * a[13],
+            b[8] * a[2] + b[9] * a[6] + b[10] * a[10] + b[11] * a[14],
+            b[8] * a[3] + b[9] * a[7] + b[10] * a[11] + b[11] * a[15],
+            b[12] * a[0] + b[13] * a[4] + b[14] * a[8] + b[15] * a[12],
+            b[12] * a[1] + b[13] * a[5] + b[14] * a[9] + b[15] * a[13],
+            b[12] * a[2] + b[13] * a[6] + b[14] * a[10] + b[15] * a[14],
+            b[12] * a[3] + b[13] * a[7] + b[14] * a[11] + b[15] * a[15]
+        );
+    }
 }
 
 export { Matrix4 };

viewer/src/lib/splat.js/math/Quaternion.ts

@@ -19,6 +19,10 @@ class Quaternion {
 
         return this;
     }
+
+    flat(): number[] {
+        return [this.x, this.y, this.z, this.w];
+    }
 }
 
 export { Quaternion };

viewer/src/lib/splat.js/math/Vector3.ts

@@ -16,6 +16,10 @@ class Vector3 {
 
         return this;
     }
+
+    flat(): number[] {
+        return [this.x, this.y, this.z];
+    }
 }
 
 export { Vector3 };

viewer/src/lib/splat.js/renderers/WebGLRenderer.ts

@@ -1,20 +1,211 @@
 import type { Camera } from "../cameras/Camera";
+import type { PerspectiveCamera } from "../cameras/PerspectiveCamera";
 import type { Scene } from "../core/Scene";
 import type { Renderer } from "./Renderer";
 
+import { getViewMatrix } from "./webgl/utils/transformations";
+import { createWorker } from "./webgl/utils/worker";
+
+import { vertex } from "./webgl/shaders/vertex.glsl";
+import { frag } from "./webgl/shaders/frag.glsl";
+import type { Matrix4 } from "../math/Matrix4";
+
 export class WebGLRenderer implements Renderer {
     canvas: HTMLCanvasElement;
 
-    gl: WebGLRenderingContext;
-    ext: ANGLE_instanced_arrays;
+    render: (scene: Scene, camera: Camera) => void;
+    dispose: () => void;
 
     constructor(canvas?: HTMLCanvasElement) {
         this.canvas =
             canvas ?? (document.createElementNS("http://www.w3.org/1999/xhtml", "canvas") as HTMLCanvasElement);
-        this.gl = this.canvas.getContext("webgl") as WebGLRenderingContext;
-        this.ext = this.gl.getExtension("ANGLE_instanced_arrays") as ANGLE_instanced_arrays;
-    }
 
-    render(scene: Scene, camera: Camera): void {}
-    dispose(): void {}
+        const gl = (this.canvas.getContext("webgl") ||
+            this.canvas.getContext("experimental-webgl")) as WebGLRenderingContext;
+
+        let ext: ANGLE_instanced_arrays;
+        let worker: Worker;
+        let vertexShader: WebGLShader;
+        let fragmentShader: WebGLShader;
+        let program: WebGLProgram;
+
+        let u_projection: WebGLUniformLocation;
+        let u_viewport: WebGLUniformLocation;
+        let u_focal: WebGLUniformLocation;
+        let u_view: WebGLUniformLocation;
+
+        let vertexLocation: number;
+        let centerLocation: number;
+        let colorLocation: number;
+        let covALocation: number;
+        let covBLocation: number;
+
+        let vertexBuffer: WebGLBuffer;
+        let centerBuffer: WebGLBuffer;
+        let colorBuffer: WebGLBuffer;
+        let covABuffer: WebGLBuffer;
+        let covBBuffer: WebGLBuffer;
+
+        function initGLContext(width: number, height: number, scene: Scene, camera: PerspectiveCamera) {
+            ext = gl.getExtension("ANGLE_instanced_arrays") as ANGLE_instanced_arrays;
+
+            worker = new Worker(
+                URL.createObjectURL(
+                    new Blob(["(", createWorker.toString(), ")(self)"], {
+                        type: "application/javascript",
+                    })
+                )
+            );
+
+            // Viewport
+            gl.viewport(0, 0, width, height);
+            camera.updateProjectionMatrix(width, height);
+
+            // Vertex shader
+            vertexShader = gl.createShader(gl.VERTEX_SHADER) as WebGLShader;
+            gl.shaderSource(vertexShader, vertex);
+            gl.compileShader(vertexShader);
+            if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
+                console.error(gl.getShaderInfoLog(vertexShader));
+            }
+
+            // Fragment shader
+            fragmentShader = gl.createShader(gl.FRAGMENT_SHADER) as WebGLShader;
+            gl.shaderSource(fragmentShader, frag);
+            gl.compileShader(fragmentShader);
+            if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
+                console.error(gl.getShaderInfoLog(fragmentShader));
+            }
+
+            // Program
+            program = gl.createProgram() as WebGLProgram;
+            gl.attachShader(program, vertexShader);
+            gl.attachShader(program, fragmentShader);
+            gl.linkProgram(program);
+            gl.useProgram(program);
+            if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+                console.error(gl.getProgramInfoLog(program));
+            }
+
+            // Blending
+            gl.disable(gl.DEPTH_TEST);
+            gl.enable(gl.BLEND);
+            gl.blendFuncSeparate(gl.ONE_MINUS_DST_ALPHA, gl.ONE, gl.ONE_MINUS_DST_ALPHA, gl.ONE);
+            gl.blendEquationSeparate(gl.FUNC_ADD, gl.FUNC_ADD);
+
+            // Uniforms
+            u_projection = gl.getUniformLocation(program, "projection") as WebGLUniformLocation;
+            gl.uniformMatrix4fv(u_projection, false, camera.projectionMatrix.buffer);
+
+            u_viewport = gl.getUniformLocation(program, "viewport") as WebGLUniformLocation;
+            gl.uniform2fv(u_viewport, new Float32Array([width, height]));
+
+            u_focal = gl.getUniformLocation(program, "focal") as WebGLUniformLocation;
+            gl.uniform2fv(u_focal, new Float32Array([camera.fx, camera.fy]));
+
+            const viewMatrix = getViewMatrix(camera);
+            u_view = gl.getUniformLocation(program, "view") as WebGLUniformLocation;
+            gl.uniformMatrix4fv(u_view, false, viewMatrix.buffer);
+
+            // Vertex buffer
+            const triangleVertices = new Float32Array([-2, -2, 2, -2, 2, 2, -2, 2]);
+            vertexBuffer = gl.createBuffer() as WebGLBuffer;
+
+            gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+            gl.bufferData(gl.ARRAY_BUFFER, triangleVertices, gl.STATIC_DRAW);
+
+            vertexLocation = gl.getAttribLocation(program, "position");
+            gl.enableVertexAttribArray(vertexLocation);
+            gl.vertexAttribPointer(vertexLocation, 2, gl.FLOAT, false, 0, 0);
+
+            // Center buffer
+            centerBuffer = gl.createBuffer() as WebGLBuffer;
+            centerLocation = gl.getAttribLocation(program, "center");
+
+            gl.enableVertexAttribArray(centerLocation);
+            gl.bindBuffer(gl.ARRAY_BUFFER, centerBuffer);
+            gl.vertexAttribPointer(centerLocation, 3, gl.FLOAT, false, 0, 0);
+            ext.vertexAttribDivisorANGLE(centerLocation, 1);
+
+            // Color buffer
+            colorBuffer = gl.createBuffer() as WebGLBuffer;
+            colorLocation = gl.getAttribLocation(program, "color");
+
+            gl.enableVertexAttribArray(colorLocation);
+            gl.bindBuffer(gl.ARRAY_BUFFER, colorBuffer);
+            gl.vertexAttribPointer(colorLocation, 4, gl.FLOAT, false, 0, 0);
+            ext.vertexAttribDivisorANGLE(colorLocation, 1);
+
+            // Covariance A buffer
+            covABuffer = gl.createBuffer() as WebGLBuffer;
+            covALocation = gl.getAttribLocation(program, "covA");
+
+            gl.enableVertexAttribArray(covALocation);
+            gl.bindBuffer(gl.ARRAY_BUFFER, covABuffer);
+            gl.vertexAttribPointer(covALocation, 3, gl.FLOAT, false, 0, 0);
+            ext.vertexAttribDivisorANGLE(covALocation, 1);
+
+            // Covariance B buffer
+            covBBuffer = gl.createBuffer() as WebGLBuffer;
+            covBLocation = gl.getAttribLocation(program, "covB");
+
+            gl.enableVertexAttribArray(covBLocation);
+            gl.bindBuffer(gl.ARRAY_BUFFER, covBBuffer);
+            gl.vertexAttribPointer(covBLocation, 3, gl.FLOAT, false, 0, 0);
+            ext.vertexAttribDivisorANGLE(covBLocation, 1);
+
+            worker.onmessage = (e) => {
+                if (e.data.center) {
+                    let { covA, covB, center, color } = e.data;
+
+                    gl.bindBuffer(gl.ARRAY_BUFFER, centerBuffer);
+                    gl.bufferData(gl.ARRAY_BUFFER, center, gl.DYNAMIC_DRAW);
+
+                    gl.bindBuffer(gl.ARRAY_BUFFER, colorBuffer);
+                    gl.bufferData(gl.ARRAY_BUFFER, color, gl.DYNAMIC_DRAW);
+
+                    gl.bindBuffer(gl.ARRAY_BUFFER, covABuffer);
+                    gl.bufferData(gl.ARRAY_BUFFER, covA, gl.DYNAMIC_DRAW);
+
+                    gl.bindBuffer(gl.ARRAY_BUFFER, covBBuffer);
+                    gl.bufferData(gl.ARRAY_BUFFER, covB, gl.DYNAMIC_DRAW);
+                }
+            };
+        }
+
+        let currentScene: Scene | null = null;
+        let currentCamera: Camera | null = null;
+
+        this.render = function (scene: Scene, camera: Camera) {
+            if (scene !== currentScene || camera !== currentCamera) {
+                const perspectiveCamera = camera as PerspectiveCamera;
+                if (perspectiveCamera == null) {
+                    throw new Error("Camera is not a PerspectiveCamera");
+                }
+
+                if (currentScene != null || currentCamera != null) {
+                    this.dispose();
+                }
+
+                currentScene = scene;
+                currentCamera = camera;
+
+                initGLContext(this.canvas.width, this.canvas.height, scene, perspectiveCamera);
+            }
+        };
+
+        this.dispose = function () {
+            gl.deleteBuffer(vertexBuffer);
+            gl.deleteBuffer(centerBuffer);
+            gl.deleteBuffer(colorBuffer);
+            gl.deleteBuffer(covABuffer);
+            gl.deleteBuffer(covBBuffer);
+
+            gl.deleteShader(vertexShader);
+            gl.deleteShader(fragmentShader);
+            gl.deleteProgram(program);
+
+            worker.terminate();
+        };
+    }
 }

viewer/src/lib/splat.js/renderers/webgl/utils/transformations.ts

@@ -0,0 +1,88 @@
+import type { Camera } from "$lib/splat.js/cameras/Camera";
+import { Vector3 } from "$lib/splat.js/math/Vector3";
+import { Matrix3 } from "$lib/splat.js/math/Matrix3";
+import { Matrix4 } from "$lib/splat.js/math/Matrix4";
+import type { Quaternion } from "$lib/splat.js/math/Quaternion";
+
+export function quaternionToEuler(q: Quaternion) {
+    return new Vector3(
+        Math.atan2(2 * (q.w * q.x + q.y * q.z), 1 - 2 * (q.x * q.x + q.y * q.y)),
+        Math.asin(2 * (q.w * q.y - q.z * q.x)),
+        Math.atan2(2 * (q.w * q.z + q.x * q.y), 1 - 2 * (q.y * q.y + q.z * q.z))
+    );
+}
+
+export function eulerToMatrix(v: Vector3) {
+    const x = v.x;
+    const y = v.y;
+    const z = v.z;
+
+    const cx = Math.cos(x);
+    const sx = Math.sin(x);
+    const cy = Math.cos(y);
+    const sy = Math.sin(y);
+    const cz = Math.cos(z);
+    const sz = Math.sin(z);
+
+    const rotationMatrix = [
+        cy * cz + sy * sx * sz,
+        -cy * sz + sy * sx * cz,
+        sy * cx,
+        cx * sz,
+        cx * cz,
+        -sx,
+        -sy * cz + cy * sx * sz,
+        sy * sz + cy * sx * cz,
+        cy * cx,
+    ];
+
+    return new Matrix4(...rotationMatrix);
+}
+
+export function quatToMatrix(q: Quaternion) {
+    const x = q.x;
+    const y = q.y;
+    const z = q.z;
+    const w = q.w;
+
+    const xx = x * x;
+    const xy = x * y;
+    const xz = x * z;
+    const xw = x * w;
+    const yy = y * y;
+    const yz = y * z;
+    const yw = y * w;
+    const zz = z * z;
+    const zw = z * w;
+
+    const rotationMatrix = [
+        1 - 2 * (yy + zz),
+        2 * (xy - zw),
+        2 * (xz + yw),
+        2 * (xy + zw),
+        1 - 2 * (xx + zz),
+        2 * (yz - xw),
+        2 * (xz - yw),
+        2 * (yz + xw),
+        1 - 2 * (xx + yy),
+    ];
+
+    return new Matrix3(...rotationMatrix);
+}
+
+export function getViewMatrix(camera: Camera) {
+    const R = quatToMatrix(camera.rotation).buffer;
+    const t = camera.position.flat();
+    const camToWorld = [
+        [R[0], R[1], R[2], 0],
+        [R[3], R[4], R[5], 0],
+        [R[6], R[7], R[8], 0],
+        [
+            -t[0] * R[0] - t[1] * R[3] - t[2] * R[6],
+            -t[0] * R[1] - t[1] * R[4] - t[2] * R[7],
+            -t[0] * R[2] - t[1] * R[5] - t[2] * R[8],
+            1,
+        ],
+    ].flat();
+    return new Matrix4(...camToWorld);
+}

viewer/src/lib/splat.js/renderers/webgl/utils/worker.ts

@@ -0,0 +1,254 @@
+export function createWorker(self: Worker) {
+    let buffer: ArrayBuffer;
+    let vertexCount = 0;
+    let viewProj: Float32Array;
+    // 6*4 + 4 + 4 = 8*4
+    // XYZ - Position (Float32)
+    // XYZ - Scale (Float32)
+    // RGBA - colors (uint8)
+    // IJKL - quaternion/rot (uint8)
+    const rowLength = 3 * 4 + 3 * 4 + 4 + 4;
+    let depthIndex = new Uint32Array();
+
+    function processPlyBuffer(inputBuffer: ArrayBuffer) {
+        const ubuf = new Uint8Array(inputBuffer);
+        // 10KB ought to be enough for a header...
+        const header = new TextDecoder().decode(ubuf.slice(0, 1024 * 10));
+        const header_end = "end_header\n";
+        const header_end_index = header.indexOf(header_end);
+        if (header_end_index < 0) throw new Error("Unable to read .ply file header");
+        const vertexCount = parseInt(/element vertex (\d+)\n/.exec(header)![1]);
+        console.log("Vertex Count", vertexCount);
+        let row_offset: number = 0;
+        let offsets: { [key: string]: number } = {};
+        let types: { [key: string]: string } = {};
+        const TYPE_MAP: { [key: string]: string } = {
+            double: "getFloat64",
+            int: "getInt32",
+            uint: "getUint32",
+            float: "getFloat32",
+            short: "getInt16",
+            ushort: "getUint16",
+            uchar: "getUint8",
+        };
+        for (let prop of header
+            .slice(0, header_end_index)
+            .split("\n")
+            .filter((k) => k.startsWith("property "))) {
+            const [_p, type, name] = prop.split(" ");
+            const arrayType = TYPE_MAP[type] || "getInt8";
+            types[name] = arrayType;
+            offsets[name] = row_offset;
+            row_offset += parseInt(arrayType.replace(/[^\d]/g, "")) / 8;
+        }
+
+        let dataView = new DataView(inputBuffer, header_end_index + header_end.length);
+        let row: number = 0;
+        const attrs: any = new Proxy(
+            {},
+            {
+                get(_target, prop: string) {
+                    if (!types[prop]) throw new Error(prop + " not found");
+                    const type = types[prop] as keyof DataView;
+                    const dataViewMethod = dataView[type] as any;
+                    return dataViewMethod(row * row_offset + offsets[prop], true);
+                },
+            }
+        );
+
+        // 6*4 + 4 + 4 = 8*4
+        // XYZ - Position (Float32)
+        // XYZ - Scale (Float32)
+        // RGBA - colors (uint8)
+        // IJKL - quaternion/rot (uint8)
+        const rowLength = 3 * 4 + 3 * 4 + 4 + 4;
+        const buffer = new ArrayBuffer(rowLength * vertexCount);
+
+        for (let j = 0; j < vertexCount; j++) {
+            row = j;
+
+            const position = new Float32Array(buffer, j * rowLength, 3);
+            const scales = new Float32Array(buffer, j * rowLength + 4 * 3, 3);
+            const rgba = new Uint8ClampedArray(buffer, j * rowLength + 4 * 3 + 4 * 3, 4);
+            const rot = new Uint8ClampedArray(buffer, j * rowLength + 4 * 3 + 4 * 3 + 4, 4);
+
+            if (types["scale_0"]) {
+                const qlen = Math.sqrt(attrs.rot_0 ** 2 + attrs.rot_1 ** 2 + attrs.rot_2 ** 2 + attrs.rot_3 ** 2);
+
+                rot[0] = (attrs.rot_0 / qlen) * 128 + 128;
+                rot[1] = (attrs.rot_1 / qlen) * 128 + 128;
+                rot[2] = (attrs.rot_2 / qlen) * 128 + 128;
+                rot[3] = (attrs.rot_3 / qlen) * 128 + 128;
+
+                scales[0] = Math.exp(attrs.scale_0);
+                scales[1] = Math.exp(attrs.scale_1);
+                scales[2] = Math.exp(attrs.scale_2);
+            } else {
+                scales[0] = 0.01;
+                scales[1] = 0.01;
+                scales[2] = 0.01;
+
+                rot[0] = 255;
+                rot[1] = 0;
+                rot[2] = 0;
+                rot[3] = 0;
+            }
+
+            position[0] = attrs.x;
+            position[1] = attrs.y;
+            position[2] = attrs.z;
+
+            if (types["f_dc_0"]) {
+                const SH_C0 = 0.28209479177387814;
+                rgba[0] = (0.5 + SH_C0 * attrs.f_dc_0) * 255;
+                rgba[1] = (0.5 + SH_C0 * attrs.f_dc_1) * 255;
+                rgba[2] = (0.5 + SH_C0 * attrs.f_dc_2) * 255;
+            } else {
+                rgba[0] = attrs.red;
+                rgba[1] = attrs.green;
+                rgba[2] = attrs.blue;
+            }
+            if (types["opacity"]) {
+                rgba[3] = (1 / (1 + Math.exp(-attrs.opacity))) * 255;
+            } else {
+                rgba[3] = 255;
+            }
+        }
+        return buffer;
+    }
+
+    const runSort = (viewProj: Float32Array) => {
+        if (!buffer) return;
+
+        const f_buffer = new Float32Array(buffer);
+        const u_buffer = new Uint8Array(buffer);
+
+        const covA = new Float32Array(3 * vertexCount);
+        const covB = new Float32Array(3 * vertexCount);
+
+        const center = new Float32Array(3 * vertexCount);
+        const color = new Float32Array(4 * vertexCount);
+
+        let maxDepth = -Infinity;
+        let minDepth = Infinity;
+        let sizeList = new Int32Array(vertexCount);
+        for (let i = 0; i < vertexCount; i++) {
+            let depth =
+                ((viewProj[2] * f_buffer[8 * i + 0] +
+                    viewProj[6] * f_buffer[8 * i + 1] +
+                    viewProj[10] * f_buffer[8 * i + 2]) *
+                    4096) |
+                0;
+            sizeList[i] = depth;
+            if (depth > maxDepth) maxDepth = depth;
+            if (depth < minDepth) minDepth = depth;
+        }
+
+        // This is a 16 bit single-pass counting sort
+        let depthInv = (256 * 256) / (maxDepth - minDepth);
+        let counts0 = new Uint32Array(256 * 256);
+        for (let i = 0; i < vertexCount; i++) {
+            sizeList[i] = ((sizeList[i] - minDepth) * depthInv) | 0;
+            counts0[sizeList[i]]++;
+        }
+        let starts0 = new Uint32Array(256 * 256);
+        for (let i = 1; i < 256 * 256; i++) starts0[i] = starts0[i - 1] + counts0[i - 1];
+        depthIndex = new Uint32Array(vertexCount);
+        for (let i = 0; i < vertexCount; i++) depthIndex[starts0[sizeList[i]]++] = i;
+
+        for (let j = 0; j < vertexCount; j++) {
+            const i = depthIndex[j];
+
+            center[3 * j + 0] = f_buffer[8 * i + 0];
+            center[3 * j + 1] = f_buffer[8 * i + 1];
+            center[3 * j + 2] = f_buffer[8 * i + 2];
+
+            color[4 * j + 0] = u_buffer[32 * i + 24 + 0] / 255;
+            color[4 * j + 1] = u_buffer[32 * i + 24 + 1] / 255;
+            color[4 * j + 2] = u_buffer[32 * i + 24 + 2] / 255;
+            color[4 * j + 3] = u_buffer[32 * i + 24 + 3] / 255;
+
+            let scale = [f_buffer[8 * i + 3 + 0], f_buffer[8 * i + 3 + 1], f_buffer[8 * i + 3 + 2]];
+            let rot = [
+                (u_buffer[32 * i + 28 + 0] - 128) / 128,
+                (u_buffer[32 * i + 28 + 1] - 128) / 128,
+                (u_buffer[32 * i + 28 + 2] - 128) / 128,
+                (u_buffer[32 * i + 28 + 3] - 128) / 128,
+            ];
+
+            const R = [
+                1.0 - 2.0 * (rot[2] * rot[2] + rot[3] * rot[3]),
+                2.0 * (rot[1] * rot[2] + rot[0] * rot[3]),
+                2.0 * (rot[1] * rot[3] - rot[0] * rot[2]),
+
+                2.0 * (rot[1] * rot[2] - rot[0] * rot[3]),
+                1.0 - 2.0 * (rot[1] * rot[1] + rot[3] * rot[3]),
+                2.0 * (rot[2] * rot[3] + rot[0] * rot[1]),
+
+                2.0 * (rot[1] * rot[3] + rot[0] * rot[2]),
+                2.0 * (rot[2] * rot[3] - rot[0] * rot[1]),
+                1.0 - 2.0 * (rot[1] * rot[1] + rot[2] * rot[2]),
+            ];
+
+            // Compute the matrix product of S and R (M = S * R)
+            const M = [
+                scale[0] * R[0],
+                scale[0] * R[1],
+                scale[0] * R[2],
+                scale[1] * R[3],
+                scale[1] * R[4],
+                scale[1] * R[5],
+                scale[2] * R[6],
+                scale[2] * R[7],
+                scale[2] * R[8],
+            ];
+
+            covA[3 * j + 0] = M[0] * M[0] + M[3] * M[3] + M[6] * M[6];
+            covA[3 * j + 1] = M[0] * M[1] + M[3] * M[4] + M[6] * M[7];
+            covA[3 * j + 2] = M[0] * M[2] + M[3] * M[5] + M[6] * M[8];
+            covB[3 * j + 0] = M[1] * M[1] + M[4] * M[4] + M[7] * M[7];
+            covB[3 * j + 1] = M[1] * M[2] + M[4] * M[5] + M[7] * M[8];
+            covB[3 * j + 2] = M[2] * M[2] + M[5] * M[5] + M[8] * M[8];
+        }
+
+        self.postMessage({ covA, center, color, covB, viewProj }, [
+            covA.buffer,
+            center.buffer,
+            color.buffer,
+            covB.buffer,
+        ]);
+    };
+
+    const throttledSort = () => {
+        if (!sortRunning) {
+            sortRunning = true;
+            let lastView = viewProj;
+            runSort(lastView);
+            setTimeout(() => {
+                sortRunning = false;
+                if (lastView !== viewProj) {
+                    throttledSort();
+                }
+            }, 0);
+        }
+    };
+
+    let sortRunning: boolean = false;
+    self.onmessage = (e) => {
+        if (e.data.ply) {
+            vertexCount = 0;
+            runSort(viewProj);
+            buffer = processPlyBuffer(e.data.ply);
+            vertexCount = Math.floor(buffer.byteLength / rowLength);
+            postMessage({ buffer: buffer });
+        } else if (e.data.buffer) {
+            buffer = e.data.buffer;
+            vertexCount = e.data.vertexCount;
+        } else if (e.data.vertexCount) {
+            vertexCount = e.data.vertexCount;
+        } else if (e.data.view) {
+            viewProj = e.data.view;
+            throttledSort();
+        }
+    };
+}

viewer/src/routes/viewer/[slug]/+page.svelte

@@ -3,6 +3,7 @@
     import type { IViewer } from "./IViewer";
     import { BabylonViewer } from "./BabylonViewer";
     import { SplatViewer } from "./SplatViewer";
+    // import { SplatViewer } from "./SplatViewer.Legacy";
 
     export let data: {
         scene: {

viewer/src/routes/viewer/[slug]/SplatViewer.Legacy.ts

@@ -639,9 +639,9 @@ export class SplatViewer implements IViewer {
         this.vertexBuffer = this.gl.createBuffer();
         this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vertexBuffer);
         this.gl.bufferData(this.gl.ARRAY_BUFFER, triangleVertices, this.gl.STATIC_DRAW);
+
         this.a_position = this.gl.getAttribLocation(this.program, "position");
         this.gl.enableVertexAttribArray(this.a_position);
-        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vertexBuffer);
         this.gl.vertexAttribPointer(this.a_position, 2, this.gl.FLOAT, false, 0, 0);
 
         // center
@@ -675,6 +675,8 @@ export class SplatViewer implements IViewer {
         this.gl.vertexAttribPointer(this.a_covB, 3, this.gl.FLOAT, false, 0, 0);
         this.ext.vertexAttribDivisorANGLE(this.a_covB, 1); // Use the extension here
 
+        console.log(this.vertexBuffer, this.centerBuffer, this.colorBuffer, this.covABuffer, this.covBBuffer);
+
         this.worker.onmessage = (e) => {
             if (e.data.buffer) {
                 this.splatData = new Uint8Array(e.data.buffer);
@@ -916,7 +918,6 @@ export class SplatViewer implements IViewer {
         this.canvas.height = innerHeight;
         this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
 
-        const aspectRatio = this.canvas.width / this.canvas.height;
         this.projectionMatrix = getProjectionMatrix(
             this.camera.fx,
             this.camera.fy,

viewer/src/routes/viewer/[slug]/SplatViewer.ts

@@ -2,17 +2,35 @@ import type { IViewer } from "./IViewer";
 import * as SPLAT from "$lib/splat.js";
 
 export class SplatViewer implements IViewer {
+    scene: SPLAT.Scene;
+    camera: SPLAT.Camera;
     renderer: SPLAT.Renderer;
+    disposed: boolean = false;
 
     constructor(canvas: HTMLCanvasElement) {
+        this.scene = new SPLAT.Scene();
+        this.camera = new SPLAT.PerspectiveCamera();
         this.renderer = new SPLAT.WebGLRenderer(canvas);
     }
 
     async loadScene(url: string, onProgress?: (progress: number) => void) {
-        await SPLAT.Loader.LoadAsync(url, this.renderer, onProgress);
+        await SPLAT.Loader.LoadAsync(url, this.scene, onProgress);
+
+        const frame = () => {
+            this.renderer.render(this.scene, this.camera);
+
+            if (!this.disposed) {
+                requestAnimationFrame(frame);
+            }
+        };
+
+        this.disposed = false;
+        requestAnimationFrame(frame);
     }
 
-    dispose() {}
+    dispose() {
+        this.disposed = true;
+    }
 
     async capture(): Promise<string | null> {
         return null;