implement inference using llama.cpp

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "llama.cpp"]
+	path = llama.cpp
+	url = https://github.com/nopperl/llama.cpp

README.md

@@ -8,9 +8,6 @@ sdk_version: 4.16.0
 app_file: app.py
 pinned: false
 license: apache-2.0
-preload_from_hub:
-  - mistralai/Mistral-7B-Instruct-v0.2
-  - nopperl/emissions-extraction-lora
 datasets:
   - nopperl/sustainability-report-emissions-instruction-style
   - nopperl/corporate-emission-reports

app.py

@@ -1,11 +1,16 @@
+from subprocess import run
+
 import gradio as gr
+from huggingface_hub import snapshot_download
 
 from corporate_emission_reports.inference import extract_emissions
 
+run(["sh", "install-llamacpp.sh"])
+MODEL_PATH = snapshot_download("nopperl/emissions-extraction-lora-merged-GGUF")
 
 def predict(input_method, document_file, document_url):
     document_path = document_file if input_method == "File" else document_url
-    emissions = extract_emissions(document_path, "mistralai/Mistral-7B-Instruct-v0.2", lora="nopperl/emissions-extraction-lora", engine="hf", low_cpu_mem_usage=True)
+    emissions = extract_emissions(document_path, MODEL_PATH, model_name="ggml-model-q8_0.gguf")
     return emissions.model_dump_json()
 
 with open("description.md", "r") as f:
@@ -28,5 +33,5 @@ interface = gr.Interface(
         analytics_enabled=False,
         cache_examples=False,
     )
-interface.queue().launch(debug=True, share=True)
+interface.queue().launch()
 

dependencies.txt

@@ -0,0 +1 @@
+cmake

install-llamacpp.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+if [ ! -f llama.cpp/build/bin/main ]; then
+  cd llama.cpp
+  mkdir build
+  cd build
+  cmake ..
+  cmake --build . --config Release
+fi

llama.cpp

@@ -0,0 +1 @@
+Subproject commit f172de03f11465dc6c5a0fc3a22f8ec254c6832c