From: Fathi Boudra Date: Sat, 15 Aug 2026 08:35:53 +0000 (+0200) Subject: common: migrate the deprecated --mmap/--no-mmap to --load-mode (#26934) X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=0177dcc7300bad8914bb838baabce87899812491;p=pkg%2Fggml%2Fsources%2Fllama.cpp common: migrate the deprecated --mmap/--no-mmap to --load-mode (#26934) Replace the deprecated --mmap, --no-mmap, --mlock, and --direct-io flags with the unified --load-mode argument across scripts, examples, and documentation. Internal warning message and env var docs updated accordingly. Signed-off-by: Fathi Boudra --- diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 192986395..8b68851ff 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -428,13 +428,13 @@ Examples: - Use device 0: ```sh -ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm none -mg 0 --mmap +ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm none -mg 0 --load-mode auto ``` - Use multiple devices: ```sh -ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm layer --mmap +ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm layer --load-mode auto ``` *Notes:* @@ -741,13 +741,13 @@ Examples: - Use device 0: ``` -build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm none -mg 0 --mmap +build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm none -mg 0 --load-mode auto ``` - Use multiple devices: ``` -build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm layer --mmap +build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm layer --load-mode auto ``` diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index fc4d160e9..9d56638e3 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -53,7 +53,7 @@ M=gpt-oss-20b-Q4_0.gguf NDEV=4 D=HTP0,HTP1,HTP2,HTP3 P=surfing.txt scripts/snapd ... LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib -GGML_HEXAGON_NDEV=4 ./bin/llama-cli --no-mmap -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf +GGML_HEXAGON_NDEV=4 ./bin/llama-cli --load-mode none -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 --device HTP0,HTP1,HTP2,HTP3 -no-cnv -f surfing.txt ... llama_model_loader: - type f32: 289 tensors diff --git a/examples/sycl/run-llama2.sh b/examples/sycl/run-llama2.sh index 6ed2535bb..c5490a515 100755 --- a/examples/sycl/run-llama2.sh +++ b/examples/sycl/run-llama2.sh @@ -18,7 +18,7 @@ CONTEXT=4096 #support malloc device memory more than 4GB. export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 -LOAD_MODE='--mmap' +LOAD_MODE='--load-mode auto' if [ $# -gt 0 ]; then GGML_SYCL_DEVICE=$1 echo "use $GGML_SYCL_DEVICE as main GPU" diff --git a/examples/sycl/start-svr.sh b/examples/sycl/start-svr.sh index 49177ba2d..c3e1b6b99 100755 --- a/examples/sycl/start-svr.sh +++ b/examples/sycl/start-svr.sh @@ -124,7 +124,7 @@ else GPUS_SETTING="-sm ${SPLIT_MODE}" fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000" -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000 +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto --host 0.0.0.0 --port 8000" +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto --host 0.0.0.0 --port 8000 diff --git a/examples/sycl/test.sh b/examples/sycl/test.sh index b9498f49b..28c2dcb20 100755 --- a/examples/sycl/test.sh +++ b/examples/sycl/test.sh @@ -133,6 +133,6 @@ else GPUS_SETTING="-sm ${SPLIT_MODE}" fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap " -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto " +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto diff --git a/examples/sycl/win-run-llama2.bat b/examples/sycl/win-run-llama2.bat index 1f2dab8d0..8bc47887d 100644 --- a/examples/sycl/win-run-llama2.bat +++ b/examples/sycl/win-run-llama2.bat @@ -7,5 +7,5 @@ set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" :: support malloc device memory more than 4GB. set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 -set LOAD_MODE="--mmap" +set LOAD_MODE="--load-mode auto" .\build\bin\llama-completion.exe -m models\llama-2-7b.Q4_0.gguf -no-cnv -p %INPUT2% -n 400 -e -ngl 99 -s 0 %LOAD_MODE% diff --git a/examples/sycl/win-start-svr.bat b/examples/sycl/win-start-svr.bat index 807710589..474212c99 100644 --- a/examples/sycl/win-start-svr.bat +++ b/examples/sycl/win-start-svr.bat @@ -188,9 +188,9 @@ if not "%GGML_SYCL_DEVICE%"=="-1" ( set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap --host 0.0.0.0 --port 8000 +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto --host 0.0.0.0 --port 8000 set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap --host 0.0.0.0 --port 8000 +%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto --host 0.0.0.0 --port 8000 endlocal diff --git a/examples/sycl/win-test.bat b/examples/sycl/win-test.bat index cc6da4413..a7c3dbb79 100644 --- a/examples/sycl/win-test.bat +++ b/examples/sycl/win-test.bat @@ -211,9 +211,9 @@ else ( set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap +%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto endlocal diff --git a/scripts/bench-models.sh b/scripts/bench-models.sh index c24101304..205f2d6b4 100755 --- a/scripts/bench-models.sh +++ b/scripts/bench-models.sh @@ -22,8 +22,8 @@ if (( QUICK )); then fi if (( DIO )); then - ARGS_BB="${ARGS_BB} --no-mmap --direct-io" - ARGS_B="${ARGS_B} -mmp 0 -dio 1" + ARGS_BB="${ARGS_BB} --load-mode dio" + ARGS_B="${ARGS_B} --load-mode dio" fi run_model() { diff --git a/scripts/snapdragon/adb/run-bench.sh b/scripts/snapdragon/adb/run-bench.sh index bbe7146b4..eaae80a77 100755 --- a/scripts/snapdragon/adb/run-bench.sh +++ b/scripts/snapdragon/adb/run-bench.sh @@ -43,7 +43,7 @@ adb $adbserial $adbhost shell " \ cd $basedir; \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --mmap 0 -m $basedir/../gguf/$model \ + $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ubatch-size 1024 -fa 1 -ngl 99 $cli_opts $@ \ " diff --git a/scripts/snapdragon/adb/run-cli.sh b/scripts/snapdragon/adb/run-cli.sh index 48127dfa2..27a4a1419 100755 --- a/scripts/snapdragon/adb/run-cli.sh +++ b/scripts/snapdragon/adb/run-cli.sh @@ -71,7 +71,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $opflt $vmem $mbuf \ - ./$branch/bin/llama-cli --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-cli --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ctx-size 8192 --ubatch-size 1024 -fa on \ -ngl 99 --device $device $cli_opts $@ \ diff --git a/scripts/snapdragon/adb/run-completion.sh b/scripts/snapdragon/adb/run-completion.sh index 2130b9a74..30893ed29 100755 --- a/scripts/snapdragon/adb/run-completion.sh +++ b/scripts/snapdragon/adb/run-completion.sh @@ -79,7 +79,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opflt $opfuse $vmem $mbuf $mmsel $fasel \ - ./$branch/bin/llama-completion --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-completion --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ctx-size 8192 --ubatch-size 1024 -fa on \ -ngl 99 --device $device $cli_opts $@ \ diff --git a/scripts/snapdragon/adb/run-mtmd.sh b/scripts/snapdragon/adb/run-mtmd.sh index 992045cb9..65dd6ec59 100755 --- a/scripts/snapdragon/adb/run-mtmd.sh +++ b/scripts/snapdragon/adb/run-mtmd.sh @@ -62,7 +62,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $experimental $sched $opmask $profile $hmx $nhvx $ndev $mtmd_backend \ - ./$branch/bin/llama-mtmd-cli --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-mtmd-cli --load-mode none -m $basedir/../gguf/$model \ --mmproj $basedir/../gguf/$mmproj \ --image $basedir/../gguf/$image \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ diff --git a/scripts/snapdragon/windows/run-bench.ps1 b/scripts/snapdragon/windows/run-bench.ps1 index 5ee81df68..6eb656e66 100644 --- a/scripts/snapdragon/windows/run-bench.ps1 +++ b/scripts/snapdragon/windows/run-bench.ps1 @@ -43,6 +43,6 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-bench.exe" ` - --mmap 0 -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ubatch-size 1024 -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-cli.ps1 b/scripts/snapdragon/windows/run-cli.ps1 index b51149bec..5da8bff33 100644 --- a/scripts/snapdragon/windows/run-cli.ps1 +++ b/scripts/snapdragon/windows/run-cli.ps1 @@ -47,7 +47,7 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-cli.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ctx-size 8192 --ubatch-size 1024 -fa on ` -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-completion.ps1 b/scripts/snapdragon/windows/run-completion.ps1 index ffce8184d..08ef139b7 100644 --- a/scripts/snapdragon/windows/run-completion.ps1 +++ b/scripts/snapdragon/windows/run-completion.ps1 @@ -47,7 +47,7 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-completion.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ctx-size 8192 --ubatch-size 1024 -fa on ` -ngl 99 -no-cnv --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-mtmd.ps1 b/scripts/snapdragon/windows/run-mtmd.ps1 index b38fae35f..6e270ec90 100644 --- a/scripts/snapdragon/windows/run-mtmd.ps1 +++ b/scripts/snapdragon/windows/run-mtmd.ps1 @@ -60,7 +60,7 @@ if ($null -ne $env:MTMD_DEVICE) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-mtmd-cli.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --mmproj $basedir\..\..\gguf\$mmproj ` --image $basedir\..\..\gguf\$image ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 5c5e97fbc..0ee63abfb 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1178,7 +1178,7 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (use_mmap) { static std::once_flag once; std::call_once(once, [] { - LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n"); + LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --load-mode none for better performance\n"); }); } } else { diff --git a/tools/completion/README.md b/tools/completion/README.md index 3bc2d823e..833687dca 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -524,13 +524,15 @@ These options help improve the performance and memory usage of the LLaMA models. - `-t N, --threads N`: Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Using the correct number of threads can greatly improve performance. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. In some systems, it is beneficial to use a higher number of threads during batch processing than during generation. If not specified, the number of threads used for batch processing will be the same as the number of threads used for generation. -### Mlock - -- `--mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. This can improve performance but trades away some of the advantages of memory-mapping by requiring more RAM to run and potentially slowing down load times as the model loads into RAM. - -### No Memory Mapping - -- `--no-mmap`: Do not memory-map the model. By default, models are mapped into memory, which allows the system to load only the necessary parts of the model as needed. However, if the model is larger than your total amount of RAM or if your system is low on available memory, using mmap might increase the risk of pageouts, negatively impacting performance. Disabling mmap results in slower load times but may reduce pageouts if you're not using `--mlock`. Note that if the model is larger than the total amount of RAM, turning off mmap would prevent the model from loading at all. +### Model Loading Mode + +- `-lm MODE, --load-mode MODE`: Specify the model loading mode (default: `auto`). + - `auto`: Memory-map the model, unless the device does not support it. + - `none`: No special loading mode. Disabling mmap results in slower load times but may reduce pageouts if you're not using `mlock`. Note that if the model is larger than the total amount of RAM, turning off mmap would prevent the model from loading at all. + - `mmap`: Memory-map the model. + - `mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. This can improve performance but trades away some of the advantages of memory-mapping by requiring more RAM to run and potentially slowing down load times as the model loads into RAM. + - `mmap+mlock`: Memory-map the model and lock it in memory. + - `dio`: Use DirectIO if available. ### NUMA support diff --git a/tools/llama-bench/README.md b/tools/llama-bench/README.md index d53978548..42cb14859 100644 --- a/tools/llama-bench/README.md +++ b/tools/llama-bench/README.md @@ -67,8 +67,8 @@ test parameters: -nkvo, --no-kv-offload <0|1> (default: 0) -fa, --flash-attn (default: auto) -dev, --device (default: auto) - -mmp, --mmap <0|1> (default: 1) - -dio, --direct-io <0|1> (default: 0) + -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode) + -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode) -embd, --embeddings <0|1> (default: 0) -ts, --tensor-split (default: 0) -ot --override-tensor =;... diff --git a/tools/server/README.md b/tools/server/README.md index ec3a33e0c..c3cbf7be0 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -296,10 +296,17 @@ For the full list of features, please refer to [server's changelog](https://gith Note: If both command line argument and environment variable are both set for the same param, the argument will take precedence over env var. -For boolean options like `--mmap` or `--kv-offload`, the environment variable is handled as shown in this example: -- `LLAMA_ARG_MMAP=true` means enabled, other accepted values are: `1`, `on`, `enabled` -- `LLAMA_ARG_MMAP=false` means disabled, other accepted values are: `0`, `off`, `disabled` -- If `LLAMA_ARG_NO_MMAP` is present (no matter the value), it means disabling mmap +For string options like `--load-mode`, the environment variable is handled as shown in this example: +- `LLAMA_ARG_LOAD_MODE=auto` sets the loading mode to auto (default) +- `LLAMA_ARG_LOAD_MODE=none` disables special loading +- `LLAMA_ARG_LOAD_MODE=mmap` enables memory-mapping +- `LLAMA_ARG_LOAD_MODE=mlock` locks the model in RAM +- `LLAMA_ARG_LOAD_MODE=mmap+mlock` enables memory-mapping and locks in RAM +- `LLAMA_ARG_LOAD_MODE=dio` uses DirectIO if available + +For boolean options like `--kv-offload`: +- `LLAMA_ARG_KV_OFFLOAD=true` means enabled, other accepted values are: `1`, `on`, `enabled` +- `LLAMA_ARG_KV_OFFLOAD=false` means disabled, other accepted values are: `0`, `off`, `disabled` Example usage of docker compose with environment variables: @@ -1893,7 +1900,7 @@ Example events: } // note for "loading" status: // - subsequent events will follow the same order of "stages" list -// - mmap is may report incorrect progress on some platforms; if you need exact progress, use --no-mmap +// - mmap may report incorrect progress on some platforms; if you need exact progress, use --load-mode none { "model": "...",