Spaces:
Paused
Paused
add another parameters
Browse files- entrypoint.sh +23 -1
entrypoint.sh
CHANGED
@@ -7,5 +7,27 @@ else
|
|
7 |
echo "The HF_TOKEN environment variable is not set or empty, not logging to Hugging Face."
|
8 |
fi
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
# Run the provided command
|
11 |
-
exec python3 -u -m vllm.entrypoints.openai.api_server
|
|
|
|
|
|
|
|
|
|
7 |
echo "The HF_TOKEN environment variable is not set or empty, not logging to Hugging Face."
|
8 |
fi
|
9 |
|
10 |
+
additional_args=${EXTRA_ARGS:-""}
|
11 |
+
if [[ ! -z "${QUANTIZATION}" ]]; then
|
12 |
+
if [[ -z "${DTYPE}" ]]; then
|
13 |
+
echo "Missing required environment variable DTYPE when QUANTIZATION is set"
|
14 |
+
exit 1
|
15 |
+
else
|
16 |
+
additional_args="${additional_args} -q ${QUANTIZATION} --dtype ${DTYPE}"
|
17 |
+
fi
|
18 |
+
fi
|
19 |
+
|
20 |
+
if [[ ! -z "${GPU_MEMORY_UTILIZATION}" ]]; then
|
21 |
+
additional_args="${additional_args} --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}"
|
22 |
+
fi
|
23 |
+
|
24 |
+
if [[ ! -z "${MAX_MODEL_LEN}" ]]; then
|
25 |
+
additional_args="${additional_args} --max-model-len ${MAX_MODEL_LEN}"
|
26 |
+
fi
|
27 |
+
|
28 |
# Run the provided command
|
29 |
+
exec python3 -u -m vllm.entrypoints.openai.api_server \
|
30 |
+
--model "${HF_MODEL}" \
|
31 |
+
--host 0.0.0.0 \
|
32 |
+
--port 7860 \
|
33 |
+
${additional_args}
|