File size: 11,810 Bytes
cf07b75
1
{"version":3,"file":"LlamaCppService-Bqc2roDm.js","sources":["../../../.svelte-kit/adapter-node/chunks/LlamaCppService.js"],"sourcesContent":["import { existsSync, mkdirSync, writeFileSync } from \"fs\";\nimport { resolve } from \"path\";\nimport { d as private_env } from \"./shared-server.js\";\nclass LlamaCppService {\n  url = \"\";\n  constructor(url) {\n    this.url = url;\n  }\n  async health() {\n    const r = await fetch(`${this.url}/health`, {\n      method: \"GET\",\n      headers: {\n        \"Content-Type\": \"application/json\"\n      }\n    });\n    const data = await r.json();\n    if (data.status === \"ok\" || data.status === \"error\" || data.status === \"loading model\") {\n      return data.status;\n    }\n    return \"unavailable\";\n  }\n  async tokenize(prompt, abortController) {\n    const response = await fetch(`${this.url}/tokenize`, {\n      method: \"POST\",\n      headers: {\n        \"Content-Type\": \"application/json\"\n      },\n      body: JSON.stringify({\n        \"content\": prompt\n      }),\n      signal: abortController.signal\n    });\n    if (response.ok) {\n      const data = await response.json();\n      if (data.tokens) {\n        return data.tokens;\n      }\n    } else if (response.status === 404) {\n      console.log(\"Tokenization endpoint not found (404).\");\n    } else {\n      console.log(`Failed to tokenize: ${await response.text()}`);\n    }\n    return null;\n  }\n  async detokenize(tokens, abortController) {\n    const response = await fetch(`${this.url}/detokenize`, {\n      method: \"POST\",\n      headers: {\n        \"Content-Type\": \"application/json\"\n      },\n      body: JSON.stringify({\n        \"tokens\": tokens\n      }),\n      signal: abortController.signal\n    });\n    if (response.ok) {\n      const data = await response.json();\n      if (data.content) {\n        return data.content.trim();\n      }\n    } else if (response.status === 404) {\n      console.log(\"Detokenization endpoint not found (404).\");\n    } else {\n      console.log(`Failed to detokenize: ${await response.text()}`);\n    }\n    return null;\n  }\n  predict(prompt, { abortController }) {\n    return async ({ prompt: prompt2 }) => {\n      console.log(prompt2.length);\n      let tokens = await this.tokenize(prompt2, abortController);\n      if (tokens !== null) {\n        console.log(\"tokens: \" + tokens.length);\n        tokens = tokens.slice(0, 30700);\n        const detokenizedPrompt = await this.detokenize(tokens, abortController);\n        if (detokenizedPrompt !== null) {\n          prompt2 = detokenizedPrompt;\n        } else {\n          prompt2 = prompt2.substring(0, 32768);\n        }\n      } else {\n        prompt2 = prompt2.substring(0, 32768);\n      }\n      console.log(prompt2.length);\n      const r = await fetch(`${this.url}/completion`, {\n        method: \"POST\",\n        headers: {\n          \"Content-Type\": \"application/json\",\n          \"Accept\": \"text/event-stream\"\n        },\n        body: JSON.stringify({\n          \"stream\": true,\n          \"n_predict\": 2e3,\n          \"temperature\": 0.2,\n          \"stop\": [\"</s>\", \"bot:\", \"user:\"],\n          // \"repeat_last_n\": 256,\n          \"repeat_penalty\": 1,\n          \"top_k\": 40,\n          \"top_p\": 0.95,\n          \"min_p\": 0.05,\n          \"seed\": 42,\n          \"n_keep\": 0,\n          // \"tfs_z\": 1,\n          // \"typical_p\": 1,\n          // \"presence_penalty\": 0,\n          // \"frequency_penalty\": 0,\n          // \"mirostat\": 0,\n          // \"mirostat_tau\": 5,\n          // \"mirostat_eta\": 0.1,\n          // // \"grammar\": \"\", \n          // \"n_probs\": 0,\n          // \"image_data\": [], \n          \"cache_prompt\": false,\n          // \"api_key\": \"\", \n          // \"slot_id\": 0, \n          \"prompt\": \"[INST]\" + prompt2 + \"[/INST]\"\n        }),\n        signal: abortController.signal\n      });\n      if (!r.ok) {\n        throw new Error(`Failed to generate text: ${await r.text()}`);\n      }\n      const encoder = new TextDecoderStream();\n      const reader = r.body?.pipeThrough(encoder).getReader();\n      let t = this;\n      return async function* () {\n        let stop = false;\n        let generatedText = \"\";\n        let tokenId = 0;\n        while (!stop) {\n          const out = await reader?.read() ?? { done: false, value: void 0 };\n          if (out.done) {\n            reader?.cancel();\n            t.createLogFile(`Request:\n${prompt2} \n\nResponse:\n${generatedText}`);\n            return;\n          }\n          if (!out.value) {\n            reader?.cancel();\n            return;\n          }\n          let tokenValue = \"\";\n          if (out.value.startsWith(\"data: \")) {\n            try {\n              let data2 = JSON.parse(out.value.slice(6));\n              tokenValue = data2.content;\n            } catch (e) {\n            }\n          } else if (private_env.LLM_API_VERSION == \"v1\") {\n            tokenValue = out.value;\n          }\n          generatedText += tokenValue.replace(\"</s>\", \"\") ?? \"\";\n          yield {\n            token: {\n              id: tokenId++,\n              text: tokenValue.replace(\"</s>\", \"\") ?? \"\",\n              logprob: 0,\n              special: false\n            },\n            generated_text: null,\n            details: null\n          };\n        }\n      }();\n    };\n  }\n  createLogFile(text) {\n    const logsDirectory = resolve(\"/logs\");\n    if (!existsSync(logsDirectory)) {\n      mkdirSync(logsDirectory);\n    }\n    const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, \"\");\n    const logFilePath = resolve(logsDirectory, `${timestamp}.log`);\n    writeFileSync(logFilePath, text);\n    console.log(`Log file created: ${logFilePath}`);\n  }\n}\nexport {\n  LlamaCppService as L\n};\n"],"names":[],"mappings":";;;;AAGA,MAAM,eAAe,CAAC;AACtB,EAAE,GAAG,GAAG,EAAE,CAAC;AACX,EAAE,WAAW,CAAC,GAAG,EAAE;AACnB,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;AACnB,GAAG;AACH,EAAE,MAAM,MAAM,GAAG;AACjB,IAAI,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;AAChD,MAAM,MAAM,EAAE,KAAK;AACnB,MAAM,OAAO,EAAE;AACf,QAAQ,cAAc,EAAE,kBAAkB;AAC1C,OAAO;AACP,KAAK,CAAC,CAAC;AACP,IAAI,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAChC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,eAAe,EAAE;AAC5F,MAAM,OAAO,IAAI,CAAC,MAAM,CAAC;AACzB,KAAK;AACL,IAAI,OAAO,aAAa,CAAC;AACzB,GAAG;AACH,EAAE,MAAM,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE;AAC1C,IAAI,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;AACzD,MAAM,MAAM,EAAE,MAAM;AACpB,MAAM,OAAO,EAAE;AACf,QAAQ,cAAc,EAAE,kBAAkB;AAC1C,OAAO;AACP,MAAM,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;AAC3B,QAAQ,SAAS,EAAE,MAAM;AACzB,OAAO,CAAC;AACR,MAAM,MAAM,EAAE,eAAe,CAAC,MAAM;AACpC,KAAK,CAAC,CAAC;AACP,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE;AACrB,MAAM,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE;AACvB,QAAQ,OAAO,IAAI,CAAC,MAAM,CAAC;AAC3B,OAAO;AACP,KAAK,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;AACxC,MAAM,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;AAC5D,KAAK,MAAM;AACX,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,oBAAoB,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAClE,KAAK;AACL,IAAI,OAAO,IAAI,CAAC;AAChB,GAAG;AACH,EAAE,MAAM,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE;AAC5C,IAAI,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE;AAC3D,MAAM,MAAM,EAAE,MAAM;AACpB,MAAM,OAAO,EAAE;AACf,QAAQ,cAAc,EAAE,kBAAkB;AAC1C,OAAO;AACP,MAAM,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;AAC3B,QAAQ,QAAQ,EAAE,MAAM;AACxB,OAAO,CAAC;AACR,MAAM,MAAM,EAAE,eAAe,CAAC,MAAM;AACpC,KAAK,CAAC,CAAC;AACP,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE;AACrB,MAAM,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;AACxB,QAAQ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;AACnC,OAAO;AACP,KAAK,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;AACxC,MAAM,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;AAC9D,KAAK,MAAM;AACX,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,sBAAsB,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACpE,KAAK;AACL,IAAI,OAAO,IAAI,CAAC;AAChB,GAAG;AACH,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,EAAE;AACvC,IAAI,OAAO,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;AAC1C,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAClC,MAAM,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;AACjE,MAAM,IAAI,MAAM,KAAK,IAAI,EAAE;AAC3B,QAAQ,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AAChD,QAAQ,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACxC,QAAQ,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;AACjF,QAAQ,IAAI,iBAAiB,KAAK,IAAI,EAAE;AACxC,UAAU,OAAO,GAAG,iBAAiB,CAAC;AACtC,SAAS,MAAM;AACf,UAAU,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAChD,SAAS;AACT,OAAO,MAAM;AACb,QAAQ,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC9C,OAAO;AACP,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAClC,MAAM,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE;AACtD,QAAQ,MAAM,EAAE,MAAM;AACtB,QAAQ,OAAO,EAAE;AACjB,UAAU,cAAc,EAAE,kBAAkB;AAC5C,UAAU,QAAQ,EAAE,mBAAmB;AACvC,SAAS;AACT,QAAQ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;AAC7B,UAAU,QAAQ,EAAE,IAAI;AACxB,UAAU,WAAW,EAAE,GAAG;AAC1B,UAAU,aAAa,EAAE,GAAG;AAC5B,UAAU,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;AAC3C;AACA,UAAU,gBAAgB,EAAE,CAAC;AAC7B,UAAU,OAAO,EAAE,EAAE;AACrB,UAAU,OAAO,EAAE,IAAI;AACvB,UAAU,OAAO,EAAE,IAAI;AACvB,UAAU,MAAM,EAAE,EAAE;AACpB,UAAU,QAAQ,EAAE,CAAC;AACrB;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,UAAU,cAAc,EAAE,KAAK;AAC/B;AACA;AACA,UAAU,QAAQ,EAAE,QAAQ,GAAG,OAAO,GAAG,SAAS;AAClD,SAAS,CAAC;AACV,QAAQ,MAAM,EAAE,eAAe,CAAC,MAAM;AACtC,OAAO,CAAC,CAAC;AACT,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;AACjB,QAAQ,MAAM,IAAI,KAAK,CAAC,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACtE,OAAO;AACP,MAAM,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;AAC9C,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC;AAC9D,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;AACnB,MAAM,OAAO,mBAAmB;AAChC,QAAQ,IAAI,IAAI,GAAG,KAAK,CAAC;AACzB,QAAQ,IAAI,aAAa,GAAG,EAAE,CAAC;AAC/B,QAAQ,IAAI,OAAO,GAAG,CAAC,CAAC;AACxB,QAAQ,OAAO,CAAC,IAAI,EAAE;AACtB,UAAU,MAAM,GAAG,GAAG,MAAM,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;AAC7E,UAAU,IAAI,GAAG,CAAC,IAAI,EAAE;AACxB,YAAY,MAAM,EAAE,MAAM,EAAE,CAAC;AAC7B,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC;AAC7B,EAAE,OAAO,CAAC;AACV;AACA;AACA,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;AACnB,YAAY,OAAO;AACnB,WAAW;AACX,UAAU,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE;AAC1B,YAAY,MAAM,EAAE,MAAM,EAAE,CAAC;AAC7B,YAAY,OAAO;AACnB,WAAW;AACX,UAAU,IAAI,UAAU,GAAG,EAAE,CAAC;AAC9B,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;AAC9C,YAAY,IAAI;AAChB,cAAc,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,cAAc,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC;AACzC,aAAa,CAAC,OAAO,CAAC,EAAE;AACxB,aAAa;AACb,WAAW,MAAM,IAAI,WAAW,CAAC,eAAe,IAAI,IAAI,EAAE;AAC1D,YAAY,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC;AACnC,WAAW;AACX,UAAU,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;AAChE,UAAU,MAAM;AAChB,YAAY,KAAK,EAAE;AACnB,cAAc,EAAE,EAAE,OAAO,EAAE;AAC3B,cAAc,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE;AACxD,cAAc,OAAO,EAAE,CAAC;AACxB,cAAc,OAAO,EAAE,KAAK;AAC5B,aAAa;AACb,YAAY,cAAc,EAAE,IAAI;AAChC,YAAY,OAAO,EAAE,IAAI;AACzB,WAAW,CAAC;AACZ,SAAS;AACT,OAAO,EAAE,CAAC;AACV,KAAK,CAAC;AACN,GAAG;AACH,EAAE,aAAa,CAAC,IAAI,EAAE;AACtB,IAAI,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE;AACpC,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC;AAC/B,KAAK;AACL,IAAI,MAAM,SAAS,GAAG,iBAAiB,IAAI,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;AACtF,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AACnE,IAAI,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;AACrC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACpD,GAAG;AACH;;;;"}