Running a GPU-accelerated LLM on FreeBSD (2-line howto)

Alain De Vos said:
The associate launcher with downloader above,

Python: 
cat myzsh
#!/bin/sh

# I was created using A.I.
# I Run on FreeBSD
# System Specs: 12GB VRAM (No Spillover), 32GB System RAM

# ==========================================
# CONFIGURATION & PARAMETERS
# ==========================================
BASE_DIR="/home/x/hugging"
HOST="127.0.0.1"
PORT="8123"

# ==========================================
# SEPARATE UTILITY FUNCTIONS
# ==========================================

# Cleanly terminate existing llama-server instances
kill_existing_server() {
    echo "Stopping any existing llama-server instances..."
    killall llama-server >/dev/null 2>&1
    sleep 2
    killall -9 llama-server >/dev/null 2>&1
    echo "Sleeping 6 seconds for port cleanup..."
    sleep 6
    echo "Done sleeping."
}

# Prompt user for the VRAM Tier directory
select_vram_tier() {
    while true; do
        printf "Select VRAM Tier Profile (8, 10, 12, 14): "
        read -r TIER
        case "$TIER" in
            8|10|12|14)
                TARGET_DIR="${BASE_DIR}/${TIER}"
                if [ ! -d "$TARGET_DIR" ]; then
                    echo "❌ Directory $TARGET_DIR does not exist."
                    continue
                fi
                return 0
                ;;
            *)
                echo "Invalid profile tier choice. Choose 8, 10, 12, or 14."
                ;;
        esac
    done
}

# Scan directory and choose an available model
select_model_file() {
    echo ""
    echo "Searching available models in: $TARGET_DIR"
    echo "--------------------------------------------------------"
  
    count=0
    set --
  
    for file in "$TARGET_DIR"/*.gguf; do
        if [ -f "$file" ]; then
            count=$((count + 1))
            set -- "$@" "$file"
            filename=$(basename "$file")
            filesize=$(du -h "$file" | cut -f1)
            echo "  $count) $filename ($filesize)"
        fi
    done

    if [ "$count" -eq 0 ]; then
        echo "❌ No GGUF binaries found inside $TARGET_DIR"
        exit 1
    fi

    while true; do
        printf "\nPlease choose a model [1-$count]: "
        read -r choice
        if echo "$choice" | egrep -q '^[0-9]+$' && [ "$choice" -ge 1 ] && [ "$choice" -le "$count" ]; then
            eval "MODEL_PATH=\${$choice}"
            return 0
        else
            echo "Invalid selection choice. Try again."
        fi
    done
}

# Key-Value Pair Lookup Database for Specific Model Names
# Maps to: ctx_size, cache_ram, ubatch_size, threads, threads_batch, temp
lookup_model_profile() {
    case "$1" in
        # --- Heavy Tiers (~10GB+ files) ---
        *DeepSeek-Coder-V2*Q5_K_M*|*DeepSeek-Coder-V2*Q8_0*|*Qwen2.5-Coder-14B*Q8_0*)
            echo "2048 512 512 4 4 0.3"
            ;;
        # --- Mid Tiers (~8.5GB - 10GB files) ---
        *Qwen2.5-Coder-14B*Q4_K_M*|*Qwen2.5-Coder-14B*Q5_K_M*|*Qwen3-14B*Q4_K_M*|*DeepSeek-Coder-V2*IQ4_XS*|*Qwen3.6-35B-A3B*UD-IQ3_S*|*Qwen3.6-35B-A3B*UD-IQ4_XS*)
            echo "8192 2048 512 4 4 0.3"
            ;;
        # --- Light Tiers (~5GB - 8GB files) ---
        *Qwen2.5-Coder-7B*Q8_0*|*DeepSeek-Coder-6.7B*Q8_0*|*Qwen3-8B*Q4_K_M*|*Qwen2.5-Coder-7B*Q5_K_M*|*Qwen2.5-Coder-7B-instruct*)
            echo "16384 4096 1024 6 4 0.3"
            ;;
        # --- Ultra Light Tiers (< 5GB files) ---
        *DeepSeek-Coder-6.7B*Q5_K_M*|*deepseek-coder:6.7b*|*Qwen3-4B*Q4_K_M*|*Qwen2.5-Coder-3B*|*DeepSeek-R1-Distill-Qwen-1.5B*|*deepseek-r1:1.5b*)
            echo "32768 8192 1024 6 4 0.3"
            ;;
        *)
            echo "NOT_FOUND"
            ;;
    esac
}

# Fallback Key-Value Pair Database using Size Category
# Maps to: ctx_size, cache_ram, ubatch_size, threads, threads_batch, temp
lookup_size_profile() {
    FILE_SIZE_GB=$1
    if [ "$FILE_SIZE_GB" -ge 10 ]; then
        echo "2048 512 512 4 4 0.3"
    elif [ "$FILE_SIZE_GB" -ge 8 ]; then
        echo "8192 2048 512 4 4 0.3"
    elif [ "$FILE_SIZE_GB" -ge 5 ]; then
        echo "16384 4096 1024 6 4 0.3"
    else
        echo "32768 8192 1024 6 4 0.3"
    fi
}

# Allocate context bounds: Try explicit key-value first, then size profile fallback
tune_vram_memory() {
    FILENAME=$(basename "$MODEL_PATH")
    echo "🔍 Analyzing model parameter mapping for: $FILENAME"

    # Step 1: Look up by model name key
    PROFILE_DATA=$(lookup_model_profile "$FILENAME")

    # Step 2: Fallback to lookup by size category key if name isn't matched
    if [ "$PROFILE_DATA" = "NOT_FOUND" ]; then
        echo "⚠️ No explicit profile name matched. Evaluating via size category key..."
        FILE_SIZE_KB=$(stat -f %z "$MODEL_PATH")
        FILE_SIZE_GB=$((FILE_SIZE_KB / 1024 / 1024 / 1024))
        echo "📊 Dynamic calculation: ${FILE_SIZE_GB} GB footprint discovered."
        PROFILE_DATA=$(lookup_size_profile "$FILE_SIZE_GB")
    else
        echo "🎯 Profile Match: Named model pair discovered."
    fi

    # Step 3: Extract values out of the selected key-value data list
    set -- $PROFILE_DATA
    CTX_SIZE=$1
    CACHE_RAM=$2
    UBATCH_SIZE=$3
    THREADS=$4
    THREADS_BATCH=$5
    TEMP=$6
}

# Execute server process loop
launch_llama_server() {
    echo "--------------------------------------------------------"
    echo "🚀 Launching Llama Server Instance"
    echo "📌 Context Size:   $CTX_SIZE"
    echo "📌 Cache RAM:     $CACHE_RAM"
    echo "📌 Threads:       $THREADS (Batch: $THREADS_BATCH)"
    echo "📌 Temperature:   $TEMP"
    echo "📌 Model:         $(basename "$MODEL_PATH")"
    echo "--------------------------------------------------------"

    llama-server \
            --ctx-size    "$CTX_SIZE" \
            --cache-ram   "$CACHE_RAM" \
            --ubatch-size "$UBATCH_SIZE" \
            --threads-batch "$THREADS_BATCH" \
            --threads     "$THREADS" \
            --host        "$HOST" \
            --port        "$PORT" \
            --temp        "$TEMP" \
            --model       "$MODEL_PATH" \
            --no-warmup \
            --parallel 1 &
          
    sleep 5
    firefox "http://${HOST}:${PORT}"
}

# ==========================================
# MAIN EXECUTION ROUTINE
# ==========================================
main() {
    kill_existing_server
    select_vram_tier
    select_model_file
    tune_vram_memory
    launch_llama_server
}

main

Note to self, never use code where you used A.I. help which , you don't fully understand (read don't become lazy) , and have not manually verified.
Click to expand...
Noob question.
Is there some nice Debug Tool for shell scripts? Like you go line by line manually and it shows you the output (simulated preferably)?
 
@cracauer, you might like this one-liner for 12GB VRAM (no spill)

(2,048 tokens)
Code: 
llama-server --ctx-size 2048 --ubatch-size 128 --threads-batch 4 --threads 4 --host 127.0.0.1 --port 8123 --temp 0.3 --model /home/x/hugging/12/Qwen3.6-35B-A3B-UD-IQ3_S.gguf --no-warmup --jinja --parallel 1 --no-mmap --fit off --n-gpu-layers all --cache-ram 4096 --flash-attn on --batch-size 128 --cpu-moe

When i need more tokens i simply do,
Code: 
ollama run deepseek-coder-v2:16b-lite-instruct-q5_K_M
 
Yes, controlling the context size is important. Think about what you need when picking an LLM for your video card. If you need more context size you might have to use a smaller model on the same video card.

Alain, why do you use no-mmap?
 
For those that want to try AI models locally with Linuxlator and CUDA on FreeBSD have a look at cracauer@ 's other thread at this link from post #30 and upward.

I have put 11 scripts there over posts #32-#35 that does this:


mainmenu.png
modelslist.png

contextoptions.png
aiderclaude.png


/grandpa
 
first thanks for toturial
second
cracauer@ said:
let me know what your hardware is and what you want to do for a recommendation.
Click to expand...
8GB VRAM (RTX 4060)
32GB DDR4 RAM

i want fastest model i can find find for mcp use (should be very good at this) and computer use (not coding)

qwen 35b-a3b is nice at speed but stupid at computer use
 
gergeskirolos59 said:
first thanks for toturial
second

8GB VRAM (RTX 4060)
32GB DDR4 RAM

i want fastest model i can find find for mcp use (should be very good at this) and computer use (not coding)

qwen 35b-a3b is nice at speed but stupid at computer use
Click to expand...
I think there is a nemotron variant that fits 8 GB.
 
Code: 
 find /home/x/hugging | grep gguf$
/home/x/hugging/Qwen2.5-32B-Instruct-Q4_K_M.gguf
/home/x/hugging/8/Qwen2.5-Coder-7B-Instruct-Q5_K_M.gguf
/home/x/hugging/8/DeepSeek-R1-Distill-Qwen-1.5B-Q8_0.gguf
/home/x/hugging/8/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf
/home/x/hugging/10/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf
/home/x/hugging/10/Qwen2.5-Coder-7B-Instruct-Q8_0.gguf
/home/x/hugging/14/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf
/home/x/hugging/14/Qwen2.5-Coder-14B-Instruct-Q5_K_M.gguf
/home/x/hugging/Qwen3.6-27B-Fable-Fus-711-UnHeretic-NM-DAU-NEO-MAX-NEO-MTP-Q4_K_M.gguf
/home/x/hugging/16/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf
/home/x/hugging/16/Qwen3.8-27B-UD-IQ2_S.gguf
/home/x/hugging/16/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf
/home/x/hugging/16/Qwen2.5-Coder-7B-Instruct-Q8_0.gguf
/home/x/hugging/16/Qwen2.5-Coder-14B-Instruct-Q5_K_M.gguf
/home/x/hugging/16/Qwen2.5-Coder-7B-Instruct-Q6_K.gguf
/home/x/hugging/16/Qwen3.8-27B-UD-IQ3_S.gguf
/home/x/hugging/12/Qwen2.5-Coder-7B-Instruct-Q8_0.gguf
/home/x/hugging/12/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf
/home/x/hugging/12/Qwen3.6-35B-A3B-UD-IQ3_S.gguf
x@myfreebsd:/mnt/MYWINZFS/data $ ls | grep gguf
Qwen3.6-27B-Fable-Fus-711-UnHeretic-NM-DAU-NEO-MAX-NEO-MTP-Q4_K_M.gguf
Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL.gguf
Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf.aria2
 
For a rough and crude guide for how to get as much as possible out of a model take a look at this table in cracauer@ 's other thread.

Basically if you want to run a model with high compression, like IQ2, then aim for a model with 70B+ parameters. Supposedly the number of parameters trained will compensate for the level of quantization.

Another example - for a model with Q8_0 then 1-8B parameters is fine.

/grandpa
 
Code: 
Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL
Launching llama-server...
[Llama Server Debug] Spawning command: llama-server --presence-penalty 0.0 --repeat-penalty 1.0 --reasoning on --reasoning-budget -1 --chat-template-kwargs {"preserve_thinking":true} --no-mmap --fit off --cache-ram 6144 --kv-unified --no-warmup --parallel 1 --flash-attn on --threads-batch 10 --threads 10 --ctx-size 65536 --batch-size 512 --ubatch-size 512 --host 127.0.0.1 --port 8123 --jinja --verbose --n-gpu-layers 24 --temp 0.6 --cpu-moe --model /home/x/data/HUGGING/HUGGING2/Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL.gguf
 
ok my latest, trust me took me long :),
Code: 
./run_launcher
[success] elapsed time: 0 s
[success] elapsed time: 0 s
[info] running (fork) Main

=== Available Models ===
1. Qwen3.5-35B-A3B-bf16-Split1 (E)
2. Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL (D)
3. Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL (C)
4. Qwen3.6-35B-A3B-UD-Q4_K_XL (E)
5. Qwen3.6-35B-A3B-MTP-UD-Q4_K_M (D)
6. Qwen3.6-35B-A3B-MTP-UD-Q4_K_M (C)
7. Qwen3.6-35B-A3B-UD-IQ4_XS (16)
8. Qwen3.6-35B-A3B-UD-IQ4_XS (14)
9. Qwen2.5-32B-Instruct-Q4_K_M (D)
10. Qwen2.5-32B-Instruct-Q4_K_M (C)
11. Qwen3.6-27B-Fable-Fus-MTP-Q4_K_M (D)
12. Qwen3.6-27B-Fable-Fus-MTP-Q4_K_M (C)
13. Qwen3.6-35B-A3B-UD-IQ3_S (12)
14. Qwen3.8-27B-UD-IQ3_S (16)
15. Qwen2.5-Coder-14B-Instruct-Q5_K_M (16)
16. Qwen2.5-Coder-14B-Instruct-Q5_K_M (14)
17. Qwen3.8-27B-UD-IQ2_S (16)
18. Qwen2.5-Coder-14B-Instruct-Q4_K_M (16)
19. Qwen2.5-Coder-14B-Instruct-Q4_K_M (12)
20. Qwen2.5-Coder-14B-Instruct-Q4_K_M (10)
21. Qwen2.5-Coder-7B-Instruct-Q8_0 (16)
22. Qwen2.5-Coder-7B-Instruct-Q8_0 (12)
23. Qwen2.5-Coder-7B-Instruct-Q8_0 (10)
24. Qwen2.5-Coder-7B-Instruct-Q6_K (16)
25. Qwen2.5-Coder-7B-Instruct-Q5_K_M (8)
26. Qwen2.5-Coder-7B-Instruct-Q4_K_M (8)
27. DeepSeek-R1-Distill-Qwen-1.5B-Q8_0 (8)

I prefer lower quantisation. Ie q5_k_m,
Code: 
Selected Model: Qwen2.5-Coder-14B-Instruct-Q5_K_M (16)
Launching llama-server...
[Llama Server Debug] 
Spawning command: llama-server 
--presence-penalty 0.0 --repeat-penalty 1.0 --reasoning on --reasoning-budget -1 
--chat-template-kwargs {"preserve_thinking":true} --no-mmap --fit off --cache-ram 6144 
--kv-unified --no-warmup --parallel 1 --flash-attn on --threads-batch 10 
--threads 10 --ctx-size 65536 --batch-size 512 --ubatch-size 512 
--host 127.0.0.1 --port 8123 --jinja --verbose --n-gpu-layers all --temp 0.3 
--model /mnt/MYWINZFS/data/HUGGING/GUFF/16/Qwen2.5-Coder-14B-Instruct-Q5_K_M.gguf
counting 1 13:32-43
we entered second if 13-32-43
 
Interesting ,
Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL (D),
roughly 80–90% of the weights have been forcefully compressed into 4-bit integers,
but the remaining 10–20% of highly vital weights are preserved at 6-bits.
 
i found this really helpful, i have an RTX 4050 with 6GB VRAM , and i train Convolutional Neural Networks (CNNs) on it because i go to competitions with them and as a newbie freebsd user i really struggled with figuring this out :3 , it gave me an idea on how to train these CNNs
 
code to query incomplete huggingface database

Python: 
#!/usr/bin/env python
import re
from huggingface_hub import HfApi

api = HfApi()

# We zoeken direct op de term 'MTP', de eigenschap die de HF database niet in zijn metadata heeft.
print("--- 🔍 APART ZOEKEN OP DE HUGGING FACE DATABANK NAAR MTP MODELLEN ---")
print("--- ⚙️ Filtert op: GGUF | 20B - 80B | 4-bit t/m 6-bit quantisatie ---\n")

try:
    models = api.list_models(
        search="MTP",
        sort="downloads",
        limit=150  # We pakken een brede selectie om handmatig te filteren
    )
except Exception as e:
    print(f"Fout bij verbinden met Hugging Face: {e}")
    exit(1)

# Doelparameters en quants definiëren
allowed_quants = ["q4_k", "q5_k", "q6_k", "ud-q4", "ud-q5", "ud-q6", "4bit", "5bit", "6bit", "iq4"]

for model in models:
    model_id = model.modelId
    model_id_lower = model_id.lower()
   
    # --- 1. SLIMME PARAMETER SCANNING ---
    # We negeren de HF metadata-totaaltelling omdat MoE/MTP vaak fout staat geregistreerd (bijv. 3B i.p.v. 35B).
    # We extraheren het getal dat vlak voor de 'B' of 'b' staat in de naam.
    size_match = re.search(r'(\d+(?:\.\d+)?)[bb]', model_id_lower)
   
    estimated_size = None
    if size_match:
        estimated_size = float(size_match.group(1))
    else:
        # Fallback check voor bekende architecturen mocht de naam afwijken
        if "next" in model_id_lower:
            estimated_size = 80.0
           
    # Strikte controle: valt de gedetecteerde grootte binnen jouw 20B - 80B wens?
    if not estimated_size or not (20.0 <= estimated_size <= 80.0):
        continue

    # --- 2. FILE & QUANTISATIE SCANNING ---
    try:
        files = api.list_repo_files(repo_id=model_id)
        matching_files = []
       
        for file in files:
            file_lower = file.lower()
            # We zoeken uitsluitend naar lokale runtime bestanden (GGUF)
            if file_lower.endswith('.gguf'):
                # Controleren of het bestand voldoet aan de 4, 5 of 6-bit wens
                if any(q in file_lower for q in allowed_quants):
                    matching_files.append(file)
                   
        # Als er bruikbare bestanden zijn gevonden, printen we het model direct uit
        if matching_files:
            print(f"📦 MATCH: {model_id} (~{estimated_size}B parameters)")
            # Sorteer zodat handige dynamic quants (zoals XL) bovenaan komen te staan
            sorted_files = sorted(matching_files, key=lambda x: "xl" in x.lower(), reverse=True)
            for f in sorted_files[:3]:  # Maximaal 3 quants per model tonen voor overzicht
                print(f"  └── File: {f}")
            print("-" * 50)
           
    except Exception:
        # Sla repositories over die offline zijn gehaald of privé zijn gemaakt
        continue

A grep on output gave me:
Qwen3.6-27B-UD-Q5_K_XL.gguf
 
what is this CPU device from vulkan info? I mean, I only have 2 GPUs and the CPU doesnt have integrated graphic card.

Code: 
vulkaninfo --summary

WARNING: [Loader Message] Code 0 : ICD for selected physical device does not export vkGetPhysicalDeviceDisplayPropertiesKHR!
WARNING: [Loader Message] Code 0 : ICD for selected physical device does not export vkGetPhysicalDeviceDisplayPlanePropertiesKHR!
WARNING: [Loader Message] Code 0 : ICD for selected physical device does not export vkGetPhysicalDeviceDisplayPropertiesKHR!
==========
VULKANINFO
==========

Vulkan Instance Version: 1.4.360


Instance Extensions: count = 26
-------------------------------
VK_EXT_acquire_drm_display             : extension revision 1
VK_EXT_acquire_xlib_display            : extension revision 1
VK_EXT_debug_report                    : extension revision 10
VK_EXT_debug_utils                     : extension revision 2
VK_EXT_direct_mode_display             : extension revision 1
VK_EXT_display_surface_counter         : extension revision 1
VK_EXT_headless_surface                : extension revision 1
VK_EXT_surface_maintenance1            : extension revision 1
VK_EXT_swapchain_colorspace            : extension revision 5
VK_KHR_device_group_creation           : extension revision 1
VK_KHR_display                         : extension revision 23
VK_KHR_external_fence_capabilities     : extension revision 1
VK_KHR_external_memory_capabilities    : extension revision 1
VK_KHR_external_semaphore_capabilities : extension revision 1
VK_KHR_get_display_properties2         : extension revision 1
VK_KHR_get_physical_device_properties2 : extension revision 2
VK_KHR_get_surface_capabilities2       : extension revision 1
VK_KHR_portability_enumeration         : extension revision 1
VK_KHR_surface                         : extension revision 25
VK_KHR_surface_maintenance1            : extension revision 1
VK_KHR_surface_protected_capabilities  : extension revision 1
VK_KHR_wayland_surface                 : extension revision 6
VK_KHR_xcb_surface                     : extension revision 6
VK_KHR_xlib_surface                    : extension revision 6
VK_LUNARG_direct_driver_loading        : extension revision 1
VK_NV_display_stereo                   : extension revision 1

Instance Layers: count = 2
--------------------------
VK_LAYER_NV_optimus NVIDIA Optimus layer      1.4.329  version 1
VK_LAYER_NV_present NVIDIA Presentation Layer 1.4.329  version 1

Devices:
========
GPU0:
    apiVersion         = 1.4.329
    driverVersion      = 595.104.2.0
    vendorID           = 0x10de
    deviceID           = 0x27b0
    deviceType         = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU
    deviceName         = NVIDIA RTX 4000 SFF Ada Generation
    driverID           = DRIVER_ID_NVIDIA_PROPRIETARY
    driverName         = NVIDIA
    driverInfo         = 595.104.02
    conformanceVersion = 1.4.3.3
    deviceUUID         = 746df7f5-bc0a-fc86-852d-ed395f83014e
    driverUUID         = 73824e66-24af-53f6-8283-b9f0f2077b45
GPU1:
    apiVersion         = 1.4.329
    driverVersion      = 595.104.2.0
    vendorID           = 0x10de
    deviceID           = 0x2704
    deviceType         = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU
    deviceName         = NVIDIA GeForce RTX 4080
    driverID           = DRIVER_ID_NVIDIA_PROPRIETARY
    driverName         = NVIDIA
    driverInfo         = 595.104.02
    conformanceVersion = 1.4.3.3
    deviceUUID         = 761b4809-9239-38db-6af9-02540e8b9980
    driverUUID         = 73824e66-24af-53f6-8283-b9f0f2077b45
GPU2:
    apiVersion         = 1.4.354
    driverVersion      = 26.2.2
    vendorID           = 0x10005
    deviceID           = 0x0000
    deviceType         = PHYSICAL_DEVICE_TYPE_CPU
    deviceName         = llvmpipe (LLVM 19.1.7, 256 bits)
    driverID           = DRIVER_ID_MESA_LLVMPIPE
    driverName         = llvmpipe
    driverInfo         = Mesa 26.2.2 (LLVM 19.1.7)
    conformanceVersion = 1.3.1.1
    deviceUUID         = 6d657361-3236-2e32-2e32-000000000000
    driverUUID         = 6c6c766d-7069-7065-5555-494400000000

Im running with `--n-gpu-layers all`



$ llama-server \
--port 8080 \
--ctx-size (math "64 * 1024") \
--temp 0.6 \
--top-p 0.95 \
--top-k 20 \
--n-gpu-layers all \
-hf bartowski/Qwen_Qwen3.5-27B-GGUF:Q6_K_L


I get about 12 tok/s
 
Thats an easy one. Your NVRAM is 16GB . For your model MTP is broken because it is split into ram.

Try,
MATCH: ManniX-ITA/Qwen3.6-27B-A3B-Coder-MTP-GGUF
File to choose: Qwen3.6-27B-A3B-Coder-CD-Q4_K_M.gguf
It takes 14.2 GB of your VRAM
Please note the quantisation is lower bit. But it will be fast.
 
atixwasfound said:
i found this really helpful, i have an RTX 4050 with 6GB VRAM , and i train Convolutional Neural Networks (CNNs) on it because i go to competitions with them and as a newbie freebsd user i really struggled with figuring this out :3 , it gave me an idea on how to train these CNNs
Click to expand...
What competitions are we talking about ?
 
You must log in or register to reply here.
Back
Top