File size: 3,437 Bytes
793fac3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
#!/bin/bash
# Source the environment
source ~/git/llama.cpp/.venv/bin/activate
# Path to llama-quantize
QUANTIZER=~/git/llama.cpp/build/bin/llama-quantize
# Detect thread count for max performance (macOS)
THREADS=$(sysctl -n hw.logicalcpu)
echo "Detected $THREADS threads."
# Find the input file (looking for F16 or f16 in the name in the current directory)
INPUT_FILE=$(find . -maxdepth 1 -name "*[Ff]16.gguf" | head -n 1)
if [ -z "$INPUT_FILE" ]; then
echo "Error: No F16 GGUF file found in the current directory."
exit 1
fi
# Remove leading ./ for cleaner filenames
INPUT_FILE=${INPUT_FILE#./}
echo "Found input file: $INPUT_FILE"
# List of quantization types requested
TYPES=(
"IQ3_M"
"IQ3_XS"
"IQ3_XXS"
"IQ4_NL"
"IQ4_XS"
"Q3_K_L"
"Q3_K_M"
"Q3_K_S"
"Q3_K_XL"
"Q4_0"
"Q4_1"
"Q4_K_L"
"Q4_K_M"
"Q4_K_S"
"Q5_K_L"
"Q5_K_M"
"Q5_K_S"
"Q6_K"
"Q6_K_L"
"Q8_0"
)
echo "Starting batch quantization..."
echo "----------------------------------------"
for TYPE in "${TYPES[@]}"; do
# Construct output filename by replacing F16 or f16 with the quant type
# Using python to handle case-insensitive replacement safely if needed, or simple bash substitution
# Simple bash substitution for F16 and f16:
OUTPUT_FILE="${INPUT_FILE/F16/$TYPE}"
OUTPUT_FILE="${OUTPUT_FILE/f16/$TYPE}"
# If substitution didn't happen (filename matches neither), just append type
if [ "$OUTPUT_FILE" == "$INPUT_FILE" ]; then
OUTPUT_FILE="${INPUT_FILE%.gguf}-$TYPE.gguf"
fi
echo "Quantizing to $TYPE..."
"$QUANTIZER" "$INPUT_FILE" "$OUTPUT_FILE" "$TYPE" "$THREADS"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo "✅ Successfully created $OUTPUT_FILE"
# Check for file size and split if necessary (Limit: 40GB)
# 40GB in bytes = 52949672960 (using 1024^3 * 40)
LIMIT_BYTES=42949672960
FILE_SIZE=$(stat -f%z "$OUTPUT_FILE")
if [ "$FILE_SIZE" -gt "$LIMIT_BYTES" ]; then
echo "File size ($FILE_SIZE bytes) exceeds 40GB. Splitting into directory..."
# Create directory name (remove .gguf extension)
DIR_NAME="${OUTPUT_FILE%.gguf}"
mkdir -p "$DIR_NAME"
# Split tool path
SPLIT_TOOL=~/git/llama.cpp/build/bin/llama-gguf-split
echo " Splitting '$OUTPUT_FILE' into '$DIR_NAME/'..."
# Change to the new directory to run the split command
pushd "$DIR_NAME" > /dev/null
# Run split command: Flags first, then IN, then OUT prefix
"$SPLIT_TOOL" --split-max-size 40G "../$OUTPUT_FILE" "$(basename "$OUTPUT_FILE" .gguf)"
SPLIT_EXIT=$?
# Change back to original directory
popd > /dev/null
if [ $SPLIT_EXIT -eq 0 ]; then
echo "✅ Split successful. Removing original large file."
rm "$OUTPUT_FILE"
else
echo "❌ Splitting failed. Keeping original file."
fi
fi
else
echo "❌ Failed to create $OUTPUT_FILE (Error code: $EXIT_CODE)"
echo " (Note: '$TYPE' might not be a valid quantization type in this version of llama.cpp)"
fi
echo "----------------------------------------"
done
echo "Batch quantization complete."
|